perf(go): start realtime projector at latest offset
This commit is contained in:
@@ -71,7 +71,7 @@ Every received frame should become one `FrameEnvelope`.
|
|||||||
1. Stabilize ingress: connection lifecycle, protocol parser correctness, bounded backpressure, durable publish.
|
1. Stabilize ingress: connection lifecycle, protocol parser correctness, bounded backpressure, durable publish.
|
||||||
2. Stabilize event log: NATS to Kafka bridge, topic names, partition keys, retry and replay.
|
2. Stabilize event log: NATS to Kafka bridge, topic names, partition keys, retry and replay.
|
||||||
3. Simplify storage: keep only the minimal tables above, remove duplicated payloads from business tables.
|
3. Simplify storage: keep only the minimal tables above, remove duplicated payloads from business tables.
|
||||||
4. Stabilize realtime: Redis full latest raw state, MySQL lightweight snapshot and location.
|
4. Stabilize realtime: Redis full latest raw state, MySQL lightweight snapshot and location. Realtime consumers start from latest when no committed offset exists; historical rebuilds must be explicit replay jobs, not accidental backlog scans.
|
||||||
5. Stabilize history: raw and location query pagination, clear time zone behavior.
|
5. Stabilize history: raw and location query pagination, clear time zone behavior.
|
||||||
6. Stabilize metrics: idempotent daily metrics from mileage differences and raw replay.
|
6. Stabilize metrics: idempotent daily metrics from mileage differences and raw replay.
|
||||||
7. Add operations: health, readiness, metrics, lag, connection counts, deploy notes.
|
7. Add operations: health, readiness, metrics, lag, connection counts, deploy notes.
|
||||||
|
|||||||
@@ -38,6 +38,8 @@
|
|||||||
|
|
||||||
如果 stat-writer 少消费某个 raw topic,对应协议的每日里程不会进入 `vehicle_daily_mileage`。修正后可能出现短时间 Kafka lag,这是在追补历史 backlog;只要 `vehicle_stat_writes_total` 持续增长且 lag 下降,就是健康状态。
|
如果 stat-writer 少消费某个 raw topic,对应协议的每日里程不会进入 `vehicle_daily_mileage`。修正后可能出现短时间 Kafka lag,这是在追补历史 backlog;只要 `vehicle_stat_writes_total` 持续增长且 lag 下降,就是健康状态。
|
||||||
|
|
||||||
|
Realtime API 是当前态投影,默认在没有已提交 offset 时从 latest 开始消费。切换 topic 或新建 consumer group 后,不应让 realtime 追扫历史 raw backlog;需要重建当前态时,应使用明确的回放任务或手动 reset offset。
|
||||||
|
|
||||||
业务端口:
|
业务端口:
|
||||||
|
|
||||||
- GB32960 TCP:`0.0.0.0:32960`
|
- GB32960 TCP:`0.0.0.0:32960`
|
||||||
|
|||||||
@@ -167,13 +167,7 @@ func consumeKafka(ctx context.Context, logger interface {
|
|||||||
Warn(string, ...any)
|
Warn(string, ...any)
|
||||||
}, registry *metrics.Registry, updater realtimeUpdater, brokers []string) {
|
}, registry *metrics.Registry, updater realtimeUpdater, brokers []string) {
|
||||||
kafkaTopics := kafkaTopicsFromEnv()
|
kafkaTopics := kafkaTopicsFromEnv()
|
||||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
reader := kafka.NewReader(realtimeReaderConfig(brokers, kafkaTopics))
|
||||||
Brokers: brokers,
|
|
||||||
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
|
||||||
GroupTopics: kafkaTopics,
|
|
||||||
MinBytes: 1,
|
|
||||||
MaxBytes: 10e6,
|
|
||||||
})
|
|
||||||
defer reader.Close()
|
defer reader.Close()
|
||||||
logger.Info("realtime kafka consumer started", "topics", strings.Join(kafkaTopics, ","))
|
logger.Info("realtime kafka consumer started", "topics", strings.Join(kafkaTopics, ","))
|
||||||
for {
|
for {
|
||||||
@@ -189,6 +183,17 @@ func consumeKafka(ctx context.Context, logger interface {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func realtimeReaderConfig(brokers []string, kafkaTopics []string) kafka.ReaderConfig {
|
||||||
|
return kafka.ReaderConfig{
|
||||||
|
Brokers: brokers,
|
||||||
|
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
|
||||||
|
GroupTopics: kafkaTopics,
|
||||||
|
MinBytes: 1,
|
||||||
|
MaxBytes: 10e6,
|
||||||
|
StartOffset: kafka.LastOffset,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func kafkaTopicsFromEnv() []string {
|
func kafkaTopicsFromEnv() []string {
|
||||||
defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
|
defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
|
||||||
return splitCSV(env("KAFKA_TOPICS", defaultTopics))
|
return splitCSV(env("KAFKA_TOPICS", defaultTopics))
|
||||||
|
|||||||
@@ -104,6 +104,17 @@ func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRealtimeReaderConfigStartsAtLatestOffset(t *testing.T) {
|
||||||
|
cfg := realtimeReaderConfig([]string{"127.0.0.1:9092"}, []string{"vehicle.raw.go.jt808.v1"})
|
||||||
|
|
||||||
|
if cfg.StartOffset != kafka.LastOffset {
|
||||||
|
t.Fatalf("StartOffset = %d, want kafka.LastOffset %d", cfg.StartOffset, kafka.LastOffset)
|
||||||
|
}
|
||||||
|
if got := strings.Join(cfg.GroupTopics, ","); got != "vehicle.raw.go.jt808.v1" {
|
||||||
|
t.Fatalf("GroupTopics = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRealtimeAPIDoesNotExposeDuplicateMileagePointRoute(t *testing.T) {
|
func TestRealtimeAPIDoesNotExposeDuplicateMileagePointRoute(t *testing.T) {
|
||||||
source, err := os.ReadFile("main.go")
|
source, err := os.ReadFile("main.go")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
Reference in New Issue
Block a user