shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(go): start realtime projector at latest offset

Browse Source
This commit is contained in:
lingniu
2026-07-03 08:30:18 +08:00
parent 4c34c1221b
commit 97362e3205
4 changed files with 26 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
go/vehicle-gateway/cmd/realtime-api/main.go
View File

@@ -167,13 +167,7 @@ func consumeKafka(ctx context.Context, logger interface {
Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, brokers []string) {
kafkaTopics := kafkaTopicsFromEnv()
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: brokers,
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
GroupTopics: kafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
})
reader := kafka.NewReader(realtimeReaderConfig(brokers, kafkaTopics))
defer reader.Close()
logger.Info("realtime kafka consumer started", "topics", strings.Join(kafkaTopics, ","))
for {
@@ -189,6 +183,17 @@ func consumeKafka(ctx context.Context, logger interface {
}
}
func realtimeReaderConfig(brokers []string, kafkaTopics []string) kafka.ReaderConfig {
return kafka.ReaderConfig{
Brokers: brokers,
GroupID: env("KAFKA_GROUP", "go-realtime-api"),
GroupTopics: kafkaTopics,
MinBytes: 1,
MaxBytes: 10e6,
StartOffset: kafka.LastOffset,
}
}
func kafkaTopicsFromEnv() []string {
defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
return splitCSV(env("KAFKA_TOPICS", defaultTopics))