feat(go): persist realtime snapshots to mysql

go/vehicle-gateway/cmd/realtime-api/main.go

@@ -44,15 +44,11 @@ func main() {
 	repository := realtime.NewRepository(client, realtime.Config{
 		OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", 600)) * time.Second,
 	})
-	if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
-		go consumeKafka(ctx, logger, repository, brokers)
-	} else {
-		logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
-	}
 
 	mux := http.NewServeMux()
 	mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
 	closeStats := func() {}
+	var updater realtimeUpdater = repository
 	if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
 		db, err := sql.Open("mysql", dsn)
 		if err != nil {
@@ -65,6 +61,16 @@ func main() {
 			os.Exit(1)
 		}
 		closeStats = func() { _ = db.Close() }
+		snapshotWriter := realtime.NewSnapshotWriter(db)
+		if env("MYSQL_REALTIME_SNAPSHOT_ENABLED", "true") != "false" {
+			if err := snapshotWriter.EnsureSchema(ctx); err != nil {
+				_ = db.Close()
+				logger.Error("realtime snapshot mysql schema bootstrap failed", "error", err)
+				os.Exit(1)
+			}
+			updater = compositeRealtimeUpdater{primary: repository, secondary: snapshotWriter}
+			logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot")
+		}
 		mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
 		logger.Info("stats mysql query enabled")
 	} else {
@@ -76,6 +82,11 @@ func main() {
 		logger.Warn("MYSQL_DSN is empty; stats query api disabled")
 	}
 	defer closeStats()
+	if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); len(brokers) > 0 {
+		go consumeKafka(ctx, logger, updater, brokers)
+	} else {
+		logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
+	}
 	closeHistory := func() {}
 	if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
 		driver := env("TDENGINE_DRIVER", "taosWS")
@@ -131,7 +142,7 @@ func consumeKafka(ctx context.Context, logger interface {
 	Info(string, ...any)
 	Error(string, ...any)
 	Warn(string, ...any)
-}, repository *realtime.Repository, brokers []string) {
+}, updater realtimeUpdater, brokers []string) {
 	reader := kafka.NewReader(kafka.ReaderConfig{
 		Brokers:     brokers,
 		GroupID:     env("KAFKA_GROUP", "go-realtime-api"),
@@ -150,7 +161,7 @@ func consumeKafka(ctx context.Context, logger interface {
 			logger.Error("kafka fetch failed", "error", err)
 			continue
 		}
-		processRealtimeMessage(ctx, logger, repository, reader, message)
+		processRealtimeMessage(ctx, logger, updater, reader, message)
 	}
 }
 
@@ -160,6 +171,21 @@ type realtimeUpdater interface {
 	Update(context.Context, envelope.FrameEnvelope) error
 }
 
+type compositeRealtimeUpdater struct {
+	primary   realtimeUpdater
+	secondary realtimeUpdater
+}
+
+func (u compositeRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
+	if err := u.primary.Update(ctx, env); err != nil {
+		return err
+	}
+	if u.secondary == nil {
+		return nil
+	}
+	return u.secondary.Update(ctx, env)
+}
+
 type kafkaMessageCommitter interface {
 	CommitMessages(context.Context, ...kafka.Message) error
 }

go/vehicle-gateway/cmd/realtime-api/main_test.go

@@ -42,6 +42,20 @@ func TestProcessRealtimeMessageUsesUncancelledContextForUpdateAndCommit(t *testi
 	}
 }
 
+func TestCompositeRealtimeUpdaterUpdatesBothStores(t *testing.T) {
+	env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"}
+	redisUpdater := &contextCheckingRealtimeUpdater{}
+	snapshotUpdater := &contextCheckingRealtimeUpdater{}
+	updater := compositeRealtimeUpdater{primary: redisUpdater, secondary: snapshotUpdater}
+
+	if err := updater.Update(context.Background(), env); err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+	if redisUpdater.count != 1 || snapshotUpdater.count != 1 {
+		t.Fatalf("updates primary=%d secondary=%d", redisUpdater.count, snapshotUpdater.count)
+	}
+}
+
 type contextCheckingRealtimeUpdater struct {
 	ctxErr error
 	count  int