refactor(go): make raw topics the realtime source
This commit is contained in:
@@ -52,8 +52,8 @@ flowchart LR
|
|||||||
| MySQL | `vehicle_daily_mileage` | Queryable daily mileage | vin, date, protocol, daily mileage, source mileage | temporary vehicle keys, generic metric key/value rows, per-frame raw details |
|
| MySQL | `vehicle_daily_mileage` | Queryable daily mileage | vin, date, protocol, daily mileage, source mileage | temporary vehicle keys, generic metric key/value rows, per-frame raw details |
|
||||||
| MySQL | `vehicle_identity_binding` | Manual identity mapping | vin, plate, phone, device id | registration history |
|
| MySQL | `vehicle_identity_binding` | Manual identity mapping | vin, plate, phone, device id | registration history |
|
||||||
| MySQL | `jt808_registration` | JT808 registration and auth trace | phone, device id, plate, auth code, vin match state, first/latest seen | GB32960 or MQTT records |
|
| MySQL | `jt808_registration` | JT808 registration and auth trace | phone, device id, plate, auth code, vin match state, first/latest seen | GB32960 or MQTT records |
|
||||||
| Redis | `vehicle:latest:{vehicleKey}` | Latest merged realtime state | cross-protocol latest fields only | full parsed payloads and historical data |
|
| Redis | `vehicle:latest:{vin}` | Latest merged realtime state | cross-protocol latest fields only for VIN-bound vehicles | full parsed payloads, temporary identities, and historical data |
|
||||||
| Redis | `vehicle:realtime-raw:{protocol}:{vehicleKey}` | Latest full realtime protocol state | latest protocol parsed payload | historical data |
|
| Redis | `vehicle:realtime-raw:{protocol}:{vin}` | Latest full realtime protocol state | latest protocol parsed payload for VIN-bound vehicles | historical data |
|
||||||
|
|
||||||
## Event Envelope Rules
|
## Event Envelope Rules
|
||||||
|
|
||||||
@@ -91,7 +91,7 @@ Runtime metrics are exposed as Prometheus text from `/metrics`. They are operati
|
|||||||
Core counters:
|
Core counters:
|
||||||
|
|
||||||
- `vehicle_gateway_frames_total`: received protocol frames by protocol and parse status.
|
- `vehicle_gateway_frames_total`: received protocol frames by protocol and parse status.
|
||||||
- `vehicle_gateway_publish_total`: raw and unified publish results by protocol.
|
- `vehicle_gateway_publish_total`: raw publish results by protocol. Unified publish metrics appear only when the explicit compatibility stream is enabled.
|
||||||
- `vehicle_realtime_kafka_messages_total`: realtime consumer messages by topic and status.
|
- `vehicle_realtime_kafka_messages_total`: realtime consumer messages by topic and status.
|
||||||
- `vehicle_realtime_updates_total`: Redis/MySQL realtime projector updates by topic and status.
|
- `vehicle_realtime_updates_total`: Redis/MySQL realtime projector updates by topic and status.
|
||||||
- `vehicle_history_writes_total`: TDengine history writes by topic and status.
|
- `vehicle_history_writes_total`: TDengine history writes by topic and status.
|
||||||
|
|||||||
@@ -25,9 +25,9 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。
|
|||||||
| 项 | 当前值 |
|
| 项 | 当前值 |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Stream | `VEHICLE_INGEST` |
|
| Stream | `VEHICLE_INGEST` |
|
||||||
| Subjects | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1`、`vehicle.event.go.unified.v1` |
|
| Subjects | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
||||||
| Durable consumer | `vehicle-kafka-bridge` |
|
| Durable consumer | `vehicle-kafka-bridge` |
|
||||||
| 语义 | Gateway 先写 NATS;bridge 写 Kafka 成功后才 ACK NATS |
|
| 语义 | Gateway 先写 NATS raw subject;bridge 写 Kafka 成功后才 ACK NATS |
|
||||||
|
|
||||||
### Kafka
|
### Kafka
|
||||||
|
|
||||||
@@ -38,7 +38,7 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。
|
|||||||
| `vehicle.raw.go.gb32960.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
| `vehicle.raw.go.gb32960.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
||||||
| `vehicle.raw.go.jt808.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
| `vehicle.raw.go.jt808.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
||||||
| `vehicle.raw.go.yutong-mqtt.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
| `vehicle.raw.go.yutong-mqtt.v1` | 12 | NATS Kafka bridge | history、stat、realtime |
|
||||||
| `vehicle.event.go.unified.v1` | 12 | NATS Kafka bridge | 预留给统一事件消费 |
|
| `vehicle.event.go.unified.v1` | 12 | 显式打开 `PUBLISH_UNIFIED_ENABLED=true` 后才写入 | 兼容旧统一事件消费;当前最小链路不依赖 |
|
||||||
|
|
||||||
审计时 Kafka 上仍存在旧 Java/Xinda/telemetry topic,例如 `vehicle.raw.gb32960.v1`、`vehicle.raw.jt808.v1`、`vehicle.event.xinda.v1`、`vehicle.raw.telemetry-input.v1`。这些不属于当前 Go 最小链路,后续如果确认没有消费者依赖,应单独做 Kafka topic 下线清单,不在业务代码里继续引用。
|
审计时 Kafka 上仍存在旧 Java/Xinda/telemetry topic,例如 `vehicle.raw.gb32960.v1`、`vehicle.raw.jt808.v1`、`vehicle.event.xinda.v1`、`vehicle.raw.telemetry-input.v1`。这些不属于当前 Go 最小链路,后续如果确认没有消费者依赖,应单独做 Kafka topic 下线清单,不在业务代码里继续引用。
|
||||||
|
|
||||||
|
|||||||
@@ -380,7 +380,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="node bus">
|
<div class="node bus">
|
||||||
<strong>Subjects</strong>
|
<strong>Subjects</strong>
|
||||||
<p><code>vehicle.raw.go.gb32960.v1</code><br><code>vehicle.raw.go.jt808.v1</code><br><code>vehicle.raw.go.yutong-mqtt.v1</code><br><code>vehicle.event.go.unified.v1</code></p>
|
<p><code>vehicle.raw.go.gb32960.v1</code><br><code>vehicle.raw.go.jt808.v1</code><br><code>vehicle.raw.go.yutong-mqtt.v1</code></p>
|
||||||
</div>
|
</div>
|
||||||
<div class="node warning">
|
<div class="node warning">
|
||||||
<strong>Async + Retry</strong>
|
<strong>Async + Retry</strong>
|
||||||
@@ -396,7 +396,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="node bus">
|
<div class="node bus">
|
||||||
<strong>Kafka Topics</strong>
|
<strong>Kafka Topics</strong>
|
||||||
<p>与 NATS subject 同名:raw 三个协议 + unified 一个实时主题。</p>
|
<p>与 NATS subject 同名:三个协议 RAW topic。<code>vehicle.event.go.unified.v1</code> 仅作为显式兼容开关保留。</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="node warning">
|
<div class="node warning">
|
||||||
<strong>失败语义</strong>
|
<strong>失败语义</strong>
|
||||||
@@ -416,7 +416,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="node store">
|
<div class="node store">
|
||||||
<strong>realtime-api 消费器</strong>
|
<strong>realtime-api 消费器</strong>
|
||||||
<p>消费 unified topic,写 Redis 在线状态、实时合并快照、各协议 realtime-raw。</p>
|
<p>消费三个 RAW topic,写 Redis 在线状态、实时合并快照、各协议 realtime-raw。</p>
|
||||||
</div>
|
</div>
|
||||||
<div class="node api">
|
<div class="node api">
|
||||||
<strong>HTTP API</strong>
|
<strong>HTTP API</strong>
|
||||||
@@ -433,7 +433,7 @@
|
|||||||
<div class="mini-step"><b>2. 拆包</b><span>TCP 按协议提取完整帧;MQTT 直接按消息处理。</span></div>
|
<div class="mini-step"><b>2. 拆包</b><span>TCP 按协议提取完整帧;MQTT 直接按消息处理。</span></div>
|
||||||
<div class="mini-step"><b>3. 解析</b><span>生成 parsed 全量结构化数据和 fields 核心字段。</span></div>
|
<div class="mini-step"><b>3. 解析</b><span>生成 parsed 全量结构化数据和 fields 核心字段。</span></div>
|
||||||
<div class="mini-step"><b>4. 绑定 VIN</b><span>根据 VIN/phone/device_id/plate 生成 vehicle_key。</span></div>
|
<div class="mini-step"><b>4. 绑定 VIN</b><span>根据 VIN/phone/device_id/plate 生成 vehicle_key。</span></div>
|
||||||
<div class="mini-step"><b>5. 发 NATS</b><span>写 RAW subject;同时写 unified subject。</span></div>
|
<div class="mini-step"><b>5. 发 NATS</b><span>写协议 RAW subject;unified 只在兼容开关打开时写。</span></div>
|
||||||
<div class="mini-step"><b>6. 桥接 Kafka</b><span>bridge 写 Kafka 成功后 ACK NATS。</span></div>
|
<div class="mini-step"><b>6. 桥接 Kafka</b><span>bridge 写 Kafka 成功后 ACK NATS。</span></div>
|
||||||
<div class="mini-step"><b>7. 多路消费</b><span>历史、实时、统计各自消费 Kafka,互不阻塞。</span></div>
|
<div class="mini-step"><b>7. 多路消费</b><span>历史、实时、统计各自消费 Kafka,互不阻塞。</span></div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ curl -fsS http://127.0.0.1:20200/readyz
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
|
| `vehicle_gateway_active_connections` | Current TCP connections by protocol. Labels: `protocol`. |
|
||||||
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
|
| `vehicle_gateway_frames_total` | Protocol frames received and parsed by the gateway. Labels: `protocol`, `status`. |
|
||||||
| `vehicle_gateway_publish_total` | Gateway publish result for raw and unified events. Labels: `protocol`, `kind`, `status`. |
|
| `vehicle_gateway_publish_total` | Gateway publish result for raw events. Unified appears only when the compatibility stream is explicitly enabled. Labels: `protocol`, `kind`, `status`. |
|
||||||
| `vehicle_bridge_messages_total` | NATS messages fetched by the bridge. Labels: `subject`, `status`. |
|
| `vehicle_bridge_messages_total` | NATS messages fetched by the bridge. Labels: `subject`, `status`. |
|
||||||
| `vehicle_bridge_kafka_writes_total` | Bridge writes to Kafka. Labels: `topic`, `status`. |
|
| `vehicle_bridge_kafka_writes_total` | Bridge writes to Kafka. Labels: `topic`, `status`. |
|
||||||
| `vehicle_bridge_nats_acks_total` | NATS ack results after Kafka write. Labels: `subject`, `status`. |
|
| `vehicle_bridge_nats_acks_total` | NATS ack results after Kafka write. Labels: `subject`, `status`. |
|
||||||
|
|||||||
@@ -33,8 +33,8 @@
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `lingniu-go-history-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
| `lingniu-go-history-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
||||||
| `lingniu-go-stat-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
| `lingniu-go-stat-writer.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
||||||
| `lingniu-go-realtime-api.service` | `vehicle.event.go.unified.v1` |
|
| `lingniu-go-realtime-api.service` | `vehicle.raw.go.gb32960.v1`、`vehicle.raw.go.jt808.v1`、`vehicle.raw.go.yutong-mqtt.v1` |
|
||||||
| `lingniu-go-nats-kafka-bridge.service` | NATS subjects 到同名 Kafka topic;包含三条 raw.go topic 和一条 unified topic |
|
| `lingniu-go-nats-kafka-bridge.service` | NATS raw subjects 到同名 Kafka topic;`vehicle.event.go.unified.v1` 只在兼容开关打开时使用 |
|
||||||
|
|
||||||
如果 stat-writer 少消费某个 raw topic,对应协议的每日里程不会进入 `vehicle_daily_mileage`。修正后可能出现短时间 Kafka lag,这是在追补历史 backlog;只要 `vehicle_stat_writes_total` 持续增长且 lag 下降,就是健康状态。
|
如果 stat-writer 少消费某个 raw topic,对应协议的每日里程不会进入 `vehicle_daily_mileage`。修正后可能出现短时间 Kafka lag,这是在追补历史 backlog;只要 `vehicle_stat_writes_total` 持续增长且 lag 下降,就是健康状态。
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ func main() {
|
|||||||
defer closeResolver()
|
defer closeResolver()
|
||||||
registry := metrics.NewRegistry()
|
registry := metrics.NewRegistry()
|
||||||
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-gateway", nil, registry))
|
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-gateway", nil, registry))
|
||||||
|
publishUnified := envBool("PUBLISH_UNIFIED_ENABLED", false)
|
||||||
|
|
||||||
protocols := []gateway.TCPProtocol{
|
protocols := []gateway.TCPProtocol{
|
||||||
{
|
{
|
||||||
@@ -74,6 +75,7 @@ func main() {
|
|||||||
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
|
||||||
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
|
||||||
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
|
||||||
|
PublishUnified: publishUnified,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
logger.Error("build tcp server failed", "protocol", protocol.Protocol, "error", err)
|
||||||
@@ -107,6 +109,7 @@ func main() {
|
|||||||
Resolver: resolver,
|
Resolver: resolver,
|
||||||
Logger: logger,
|
Logger: logger,
|
||||||
Metrics: registry,
|
Metrics: registry,
|
||||||
|
PublishUnified: publishUnified,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("build yutong mqtt client failed", "error", err)
|
logger.Error("build yutong mqtt client failed", "error", err)
|
||||||
@@ -183,7 +186,7 @@ func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error)
|
|||||||
})
|
})
|
||||||
logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds())
|
logger.Info("nats async publish enabled", "queue_size", queueSize, "workers", workers, "publish_timeout_ms", timeout.Milliseconds())
|
||||||
}
|
}
|
||||||
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject)
|
logger.Info("nats jetstream sink enabled", "url", env("NATS_URL", ""), "unified_subject", natsSinkConfigFromEnv().UnifiedSubject, "publish_unified_enabled", envBool("PUBLISH_UNIFIED_ENABLED", false))
|
||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
|
||||||
|
|||||||
@@ -105,7 +105,9 @@ func loadConfig() config {
|
|||||||
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
|
env("NATS_SUBJECT_GB32960_RAW", env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960)): env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960),
|
||||||
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
|
env("NATS_SUBJECT_JT808_RAW", env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808)): env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808),
|
||||||
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
|
env("NATS_SUBJECT_YUTONG_MQTT_RAW", env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT)): env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT),
|
||||||
env("NATS_SUBJECT_UNIFIED", env("KAFKA_TOPIC_UNIFIED", topics.Unified)): env("KAFKA_TOPIC_UNIFIED", topics.Unified),
|
}
|
||||||
|
if unifiedSubject, unifiedTopic, ok := unifiedRouteFromEnv(); ok {
|
||||||
|
route[unifiedSubject] = unifiedTopic
|
||||||
}
|
}
|
||||||
return config{
|
return config{
|
||||||
NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),
|
NATSURL: env("NATS_URL", "nats://127.0.0.1:4222"),
|
||||||
@@ -124,6 +126,24 @@ func loadConfig() config {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func unifiedRouteFromEnv() (string, string, bool) {
|
||||||
|
subject, hasSubject := envOptional("NATS_SUBJECT_UNIFIED")
|
||||||
|
topic, hasTopic := envOptional("KAFKA_TOPIC_UNIFIED")
|
||||||
|
if !hasSubject && !hasTopic {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
if subject == "" {
|
||||||
|
subject = topic
|
||||||
|
}
|
||||||
|
if topic == "" {
|
||||||
|
topic = subject
|
||||||
|
}
|
||||||
|
if subject == "" || topic == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return subject, topic, true
|
||||||
|
}
|
||||||
|
|
||||||
type natsPullSubscription interface {
|
type natsPullSubscription interface {
|
||||||
Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
|
Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
|
||||||
}
|
}
|
||||||
@@ -290,6 +310,14 @@ func env(key string, fallback string) string {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func envOptional(key string) (string, bool) {
|
||||||
|
value, ok := os.LookupEnv(key)
|
||||||
|
if !ok {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(value), true
|
||||||
|
}
|
||||||
|
|
||||||
func envInt(key string, fallback int) int {
|
func envInt(key string, fallback int) int {
|
||||||
value := strings.TrimSpace(os.Getenv(key))
|
value := strings.TrimSpace(os.Getenv(key))
|
||||||
if value == "" {
|
if value == "" {
|
||||||
|
|||||||
@@ -120,7 +120,6 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
|
|||||||
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
|
"vehicle.raw.go.gb32960.v1": "vehicle.raw.go.gb32960.v1",
|
||||||
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
|
"vehicle.raw.go.jt808.v1": "vehicle.raw.go.jt808.v1",
|
||||||
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
|
"vehicle.raw.go.yutong-mqtt.v1": "vehicle.raw.go.yutong-mqtt.v1",
|
||||||
"vehicle.event.go.unified.v1": "vehicle.event.go.unified.v1",
|
|
||||||
}
|
}
|
||||||
if len(cfg.Route) != len(want) {
|
if len(cfg.Route) != len(want) {
|
||||||
t.Fatalf("route len = %d, want %d: %#v", len(cfg.Route), len(want), cfg.Route)
|
t.Fatalf("route len = %d, want %d: %#v", len(cfg.Route), len(want), cfg.Route)
|
||||||
@@ -132,6 +131,17 @@ func TestLoadConfigDefaultsToGoSubjectRoutes(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadConfigIncludesUnifiedOnlyWhenExplicitlyConfigured(t *testing.T) {
|
||||||
|
t.Setenv("NATS_SUBJECT_UNIFIED", "vehicle.event.go.unified.v1")
|
||||||
|
t.Setenv("KAFKA_TOPIC_UNIFIED", "vehicle.event.go.unified.v1")
|
||||||
|
|
||||||
|
cfg := loadConfig()
|
||||||
|
|
||||||
|
if got := cfg.Route["vehicle.event.go.unified.v1"]; got != "vehicle.event.go.unified.v1" {
|
||||||
|
t.Fatalf("unified route = %q, route=%#v", got, cfg.Route)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type recordingBridgeWriter struct {
|
type recordingBridgeWriter struct {
|
||||||
messages []kafka.Message
|
messages []kafka.Message
|
||||||
err error
|
err error
|
||||||
|
|||||||
@@ -190,7 +190,8 @@ func consumeKafka(ctx context.Context, logger interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func kafkaTopicsFromEnv() []string {
|
func kafkaTopicsFromEnv() []string {
|
||||||
return splitCSV(env("KAFKA_TOPICS", topics.Unified))
|
defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
|
||||||
|
return splitCSV(env("KAFKA_TOPICS", defaultTopics))
|
||||||
}
|
}
|
||||||
|
|
||||||
const kafkaMessageOperationTimeout = 30 * time.Second
|
const kafkaMessageOperationTimeout = 30 * time.Second
|
||||||
|
|||||||
@@ -66,15 +66,15 @@ func TestProcessRealtimeMessageRecordsMetrics(t *testing.T) {
|
|||||||
registry,
|
registry,
|
||||||
&contextCheckingRealtimeUpdater{},
|
&contextCheckingRealtimeUpdater{},
|
||||||
&contextCheckingMessageCommitter{},
|
&contextCheckingMessageCommitter{},
|
||||||
kafka.Message{Topic: "vehicle.event.go.unified.v1", Partition: 2, Offset: 10, HighWaterMark: 16, Value: payload},
|
kafka.Message{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 10, HighWaterMark: 16, Value: payload},
|
||||||
)
|
)
|
||||||
|
|
||||||
text := registry.Render()
|
text := registry.Render()
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
`vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.event.go.unified.v1"} 1`,
|
`vehicle_realtime_kafka_messages_total{status="received",topic="vehicle.raw.go.jt808.v1"} 1`,
|
||||||
`vehicle_realtime_updates_total{status="ok",topic="vehicle.event.go.unified.v1"} 1`,
|
`vehicle_realtime_updates_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`,
|
||||||
`vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.event.go.unified.v1"} 1`,
|
`vehicle_realtime_kafka_commits_total{status="ok",topic="vehicle.raw.go.jt808.v1"} 1`,
|
||||||
`vehicle_realtime_kafka_lag{partition="2",topic="vehicle.event.go.unified.v1"} 5`,
|
`vehicle_realtime_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 5`,
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(text, want) {
|
if !strings.Contains(text, want) {
|
||||||
t.Fatalf("metrics missing %s:\n%s", want, text)
|
t.Fatalf("metrics missing %s:\n%s", want, text)
|
||||||
@@ -96,9 +96,9 @@ func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKafkaTopicsFromEnvDefaultsToGoUnifiedTopic(t *testing.T) {
|
func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) {
|
||||||
got := strings.Join(kafkaTopicsFromEnv(), ",")
|
got := strings.Join(kafkaTopicsFromEnv(), ",")
|
||||||
want := "vehicle.event.go.unified.v1"
|
want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
|
||||||
if got != want {
|
if got != want {
|
||||||
t.Fatalf("topics = %q, want %q", got, want)
|
t.Fatalf("topics = %q, want %q", got, want)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ type MQTTClientConfig struct {
|
|||||||
Resolver identity.Resolver
|
Resolver identity.Resolver
|
||||||
Logger *slog.Logger
|
Logger *slog.Logger
|
||||||
Metrics *metrics.Registry
|
Metrics *metrics.Registry
|
||||||
|
PublishUnified bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type MQTTClient struct {
|
type MQTTClient struct {
|
||||||
@@ -220,12 +221,14 @@ func (c *MQTTClient) handleMessage(ctx context.Context, topic string, payload []
|
|||||||
if env.ParseStatus == envelope.ParseBadFrame {
|
if env.ParseStatus == envelope.ParseBadFrame {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil {
|
if c.cfg.PublishUnified {
|
||||||
c.recordPublishMetric("unified", "error")
|
if err := c.cfg.Sink.PublishUnified(messageCtx, env); err != nil {
|
||||||
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
c.recordPublishMetric("unified", "error")
|
||||||
return
|
c.cfg.Logger.Error("publish mqtt unified failed", "topic", topic, "event_id", env.StableEventID(), "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.recordPublishMetric("unified", "ok")
|
||||||
}
|
}
|
||||||
c.recordPublishMetric("unified", "ok")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
|
func (c *MQTTClient) recordFrameMetric(status envelope.ParseStatus) {
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ import (
|
|||||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
func TestMQTTClientHandleMessagePublishesOnlyRawByDefault(t *testing.T) {
|
||||||
sink := &recordingSink{}
|
sink := &recordingSink{}
|
||||||
client, err := NewMQTTClient(MQTTClientConfig{
|
client, err := NewMQTTClient(MQTTClientConfig{
|
||||||
EndpointName: "endpoint-a",
|
EndpointName: "endpoint-a",
|
||||||
@@ -39,7 +39,7 @@ func TestMQTTClientHandleMessagePublishesRawAndUnified(t *testing.T) {
|
|||||||
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||||
}`))
|
}`))
|
||||||
|
|
||||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
}
|
}
|
||||||
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
if sink.raw[0].Protocol != envelope.ProtocolYutongMQTT || sink.raw[0].VIN != "LTEST000000000001" {
|
||||||
@@ -72,12 +72,14 @@ func TestMQTTClientRecordsMessageMetrics(t *testing.T) {
|
|||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
|
`vehicle_gateway_frames_total{protocol="YUTONG_MQTT",status="OK"} 1`,
|
||||||
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
|
`vehicle_gateway_publish_total{kind="raw",protocol="YUTONG_MQTT",status="ok"} 1`,
|
||||||
`vehicle_gateway_publish_total{kind="unified",protocol="YUTONG_MQTT",status="ok"} 1`,
|
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(text, want) {
|
if !strings.Contains(text, want) {
|
||||||
t.Fatalf("metrics missing %s:\n%s", want, text)
|
t.Fatalf("metrics missing %s:\n%s", want, text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.Contains(text, `kind="unified"`) {
|
||||||
|
t.Fatalf("unified publish metric should not be recorded by default:\n%s", text)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
func TestMQTTClientHandleBadPayloadPublishesOnlyRaw(t *testing.T) {
|
||||||
@@ -138,11 +140,37 @@ func TestMQTTClientUsesUncancelledMessageContextForReceivedMessage(t *testing.T)
|
|||||||
if sink.unifiedCtxErr != nil {
|
if sink.unifiedCtxErr != nil {
|
||||||
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
|
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
|
||||||
}
|
}
|
||||||
if sink.rawCount != 1 || sink.unifiedCount != 1 {
|
if sink.rawCount != 1 || sink.unifiedCount != 0 {
|
||||||
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
|
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMQTTClientPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
|
||||||
|
sink := &recordingSink{}
|
||||||
|
client, err := NewMQTTClient(MQTTClientConfig{
|
||||||
|
EndpointName: "endpoint-a",
|
||||||
|
Broker: "tcp://127.0.0.1:1883",
|
||||||
|
ClientID: "test-client",
|
||||||
|
Topics: []string{"/ytforward/shln/+"},
|
||||||
|
PublishUnified: true,
|
||||||
|
Sink: sink,
|
||||||
|
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewMQTTClient() error = %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
client.handleMessage(context.Background(), "/ytforward/shln/dev1", []byte(`{
|
||||||
|
"device":"LTEST000000000001",
|
||||||
|
"time":"20260413100000",
|
||||||
|
"data":{"METER_SPEED":52.3,"TOTAL_MILEAGE":123456.7}
|
||||||
|
}`))
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
|
func TestMQTTClientBuildOptionsLoadsTLSCertificates(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
|
caPath, certPath, keyPath := writeTestTLSMaterial(t, dir)
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ type TCPServer struct {
|
|||||||
readBufferSize int
|
readBufferSize int
|
||||||
idleTimeout time.Duration
|
idleTimeout time.Duration
|
||||||
maxConnections int
|
maxConnections int
|
||||||
|
publishUnified bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type TCPServerConfig struct {
|
type TCPServerConfig struct {
|
||||||
@@ -52,6 +53,7 @@ type TCPServerConfig struct {
|
|||||||
ReadBufferSize int
|
ReadBufferSize int
|
||||||
IdleTimeout time.Duration
|
IdleTimeout time.Duration
|
||||||
MaxConnections int
|
MaxConnections int
|
||||||
|
PublishUnified bool
|
||||||
}
|
}
|
||||||
|
|
||||||
const frameOperationTimeout = 30 * time.Second
|
const frameOperationTimeout = 30 * time.Second
|
||||||
@@ -96,6 +98,7 @@ func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
|||||||
readBufferSize: cfg.ReadBufferSize,
|
readBufferSize: cfg.ReadBufferSize,
|
||||||
idleTimeout: cfg.IdleTimeout,
|
idleTimeout: cfg.IdleTimeout,
|
||||||
maxConnections: cfg.MaxConnections,
|
maxConnections: cfg.MaxConnections,
|
||||||
|
publishUnified: cfg.PublishUnified,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -226,12 +229,14 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
|
|||||||
if env.ParseStatus == envelope.ParseBadFrame {
|
if env.ParseStatus == envelope.ParseBadFrame {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := s.sink.PublishUnified(frameCtx, env); err != nil {
|
if s.publishUnified {
|
||||||
s.recordPublishMetric("unified", "error")
|
if err := s.sink.PublishUnified(frameCtx, env); err != nil {
|
||||||
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
s.recordPublishMetric("unified", "error")
|
||||||
return
|
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.recordPublishMetric("unified", "ok")
|
||||||
}
|
}
|
||||||
s.recordPublishMetric("unified", "ok")
|
|
||||||
if s.protocol.Respond == nil {
|
if s.protocol.Respond == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ import (
|
|||||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
func TestTCPServerPublishesGoodFrameOnlyToRawByDefault(t *testing.T) {
|
||||||
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
frame, err := hex.DecodeString("7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -36,7 +36,7 @@ func TestTCPServerPublishesGoodFrameToRawAndUnified(t *testing.T) {
|
|||||||
_ = client.Close()
|
_ = client.Close()
|
||||||
<-done
|
<-done
|
||||||
|
|
||||||
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
if len(sink.raw) != 1 || len(sink.unified) != 0 {
|
||||||
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
}
|
}
|
||||||
if sink.raw[0].Phone != "13307795425" {
|
if sink.raw[0].Phone != "13307795425" {
|
||||||
@@ -80,12 +80,14 @@ func TestTCPServerRecordsFrameMetrics(t *testing.T) {
|
|||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
|
`vehicle_gateway_frames_total{protocol="JT808",status="OK"} 1`,
|
||||||
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
|
`vehicle_gateway_publish_total{kind="raw",protocol="JT808",status="ok"} 1`,
|
||||||
`vehicle_gateway_publish_total{kind="unified",protocol="JT808",status="ok"} 1`,
|
|
||||||
} {
|
} {
|
||||||
if !strings.Contains(text, want) {
|
if !strings.Contains(text, want) {
|
||||||
t.Fatalf("metrics missing %s:\n%s", want, text)
|
t.Fatalf("metrics missing %s:\n%s", want, text)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.Contains(text, `kind="unified"`) {
|
||||||
|
t.Fatalf("unified publish metric should not be recorded by default:\n%s", text)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) {
|
func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) {
|
||||||
@@ -171,8 +173,8 @@ func TestTCPServerWritesProtocolResponseAfterPublish(t *testing.T) {
|
|||||||
Extract: gb32960.ExtractFrames,
|
Extract: gb32960.ExtractFrames,
|
||||||
Parse: gb32960.ParseFrame,
|
Parse: gb32960.ParseFrame,
|
||||||
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
Respond: func(_ []byte, env envelope.FrameEnvelope) ([]byte, bool, error) {
|
||||||
if len(sink.unified) != 1 || sink.unified[0].EventID != env.EventID {
|
if len(sink.raw) != 1 || sink.raw[0].EventID != env.EventID {
|
||||||
t.Fatalf("response built before publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
t.Fatalf("response built before raw publish: raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
}
|
}
|
||||||
return []byte("ACK"), true, nil
|
return []byte("ACK"), true, nil
|
||||||
},
|
},
|
||||||
@@ -235,11 +237,38 @@ func TestTCPServerUsesUncancelledFrameContextForAlreadyReadFrame(t *testing.T) {
|
|||||||
if sink.unifiedCtxErr != nil {
|
if sink.unifiedCtxErr != nil {
|
||||||
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
|
t.Fatalf("unified publish saw cancelled context: %v", sink.unifiedCtxErr)
|
||||||
}
|
}
|
||||||
if sink.rawCount != 1 || sink.unifiedCount != 1 {
|
if sink.rawCount != 1 || sink.unifiedCount != 0 {
|
||||||
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
|
t.Fatalf("raw=%d unified=%d", sink.rawCount, sink.unifiedCount)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTCPServerPublishesUnifiedWhenExplicitlyEnabled(t *testing.T) {
|
||||||
|
sink := &recordingSink{}
|
||||||
|
server := newTestServer(t, TCPProtocol{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
Addr: ":0",
|
||||||
|
Extract: jt808.ExtractFrames,
|
||||||
|
Parse: func(_ []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
|
||||||
|
return envelope.FrameEnvelope{
|
||||||
|
Protocol: envelope.ProtocolJT808,
|
||||||
|
MessageID: "0x0200",
|
||||||
|
Phone: "13307795425",
|
||||||
|
SourceEndpoint: sourceEndpoint,
|
||||||
|
ReceivedAtMS: receivedAtMS,
|
||||||
|
EventTimeMS: receivedAtMS,
|
||||||
|
ParseStatus: envelope.ParseOK,
|
||||||
|
}, nil
|
||||||
|
},
|
||||||
|
}, sink)
|
||||||
|
server.publishUnified = true
|
||||||
|
|
||||||
|
server.handleFrame(context.Background(), nil, []byte{0x01}, "127.0.0.1:808")
|
||||||
|
|
||||||
|
if len(sink.raw) != 1 || len(sink.unified) != 1 {
|
||||||
|
t.Fatalf("raw=%d unified=%d", len(sink.raw), len(sink.unified))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type contextCheckingResolver struct {
|
type contextCheckingResolver struct {
|
||||||
ctxErr error
|
ctxErr error
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user