lingniu-vehicle-ingest/go/vehicle-gateway/cmd/realtime-api/main.go

package main

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/redis/go-redis/v9"
	"github.com/segmentio/kafka-go"
	_ "github.com/taosdata/driver-go/v3/taosWS"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/history"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)

const (
	defaultThrottleCacheRetention       = 72 * time.Hour
	defaultThrottleCacheCleanupInterval = 10 * time.Minute
)

func main() {
	role := realtimeRoleFromEnv()
	apiEnabled := realtimeRoleAllowsAPI(role)
	writerEnabled := realtimeRoleAllowsWriter(role)
	redisProjectorEnabled := env("REALTIME_REDIS_PROJECTOR_ENABLED", "true") != "false"
	serviceName := realtimeServiceName(role)
	logger := observability.NewLogger(serviceName)
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	mux := http.NewServeMux()
	registry := metrics.NewRegistry()
	metrics.RegisterServiceInfo(registry, serviceName)
	registry.SetGauge("vehicle_realtime_role_info", metrics.Labels{"role": role}, 1)
	registry.SetGauge("vehicle_realtime_writer_enabled", nil, boolGauge(writerEnabled))
	registry.SetGauge("vehicle_realtime_redis_projector_enabled", nil, boolGauge(redisProjectorEnabled))
	healthChecks := []health.Check{}
	var client *redis.Client
	var repository *realtime.Repository
	if realtimeRoleNeedsRedis(apiEnabled, writerEnabled, redisProjectorEnabled) {
		client = redis.NewClient(&redis.Options{
			Addr:     env("REDIS_ADDR", "127.0.0.1:6379"),
			Username: env("REDIS_USERNAME", ""),
			Password: env("REDIS_PASSWORD", ""),
			DB:       envInt("REDIS_DB", 0),
		})
		defer client.Close()
		if err := client.Ping(ctx).Err(); err != nil {
			logger.Error("redis ping failed", "error", err)
			os.Exit(1)
		}
		repository = realtime.NewRepository(client, realtime.Config{
			OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", envInt("REALTIME_ONLINE_TTL_SECONDS", 60))) * time.Second,
		})
		healthChecks = append(healthChecks, health.Check{Name: "redis", Check: func(ctx context.Context) error { return client.Ping(ctx).Err() }})
	} else {
		logger.Info("redis dependency disabled by role", "role", role, "redis_projector_enabled", redisProjectorEnabled)
	}
	mux.Handle("/metrics", metrics.NewHandler(registry))
	if apiEnabled {
		mux.Handle("/openapi.json", realtime.NewOpenAPIHandler())
		mux.Handle("/swagger-ui/", realtime.NewSwaggerUIHandler())
		mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
		mux.Handle("/api/realtime/online", realtime.NewOnlineIndexHandler(repository))
		mux.Handle("/api/debug/pipeline", realtime.NewPipelineDebugHandler(repository))
	} else {
		logger.Info("realtime query api disabled by role", "role", role)
	}
	closeStats := func() {}
	closeRealtimeUpdaters := func() {}
	var redisUpdater realtimeUpdater
	if writerEnabled && redisProjectorEnabled {
		redisUpdater = storeMetricUpdater{
			store:    "redis",
			delegate: fastRealtimeUpdater{repository: repository},
			registry: registry,
		}
	} else {
		logger.Info("realtime redis kafka projector disabled; redis query api still enabled")
	}
	var updater realtimeUpdater = redisUpdater
	if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
		db, err := sql.Open("mysql", dsn)
		if err != nil {
			logger.Error("stats mysql open failed", "error", err)
			os.Exit(1)
		}
		if err := db.PingContext(ctx); err != nil {
			_ = db.Close()
			logger.Error("stats mysql ping failed", "error", err)
			os.Exit(1)
		}
		healthChecks = append(healthChecks, health.Check{Name: "mysql", Check: db.PingContext})
		closeStats = func() { _ = db.Close() }
		bindingTable := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding")
		plateCacheMaxEntries := envInt("PLATE_CACHE_MAX_ENTRIES", 200000)
		plateResolver := realtime.NewCachedPlateResolverWithMaxEntries(
			realtime.NewBindingPlateResolver(db, bindingTable),
			time.Duration(envInt("PLATE_CACHE_TTL_SECONDS", 600))*time.Second,
			plateCacheMaxEntries,
		)
		plateResolver.SetStatsObserver(func(stats realtime.PlateCacheStats) {
			recordPlateCacheStats(registry, stats)
		})
		registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(plateCacheMaxEntries))
		snapshotWriter := realtime.NewSnapshotWriterWithPlateResolver(db, plateResolver)
		if writerEnabled && env("MYSQL_REALTIME_SNAPSHOT_ENABLED", "true") != "false" {
			if err := snapshotWriter.EnsureSchema(ctx); err != nil {
				_ = db.Close()
				logger.Error("realtime snapshot mysql schema bootstrap failed", "error", err)
				os.Exit(1)
			}
			mysqlDelegate := realtimeUpdater(retryRealtimeUpdater{
				delegate: snapshotWriter,
				attempts: envInt("MYSQL_REALTIME_RETRY_ATTEMPTS", 3),
				delay:    time.Duration(envInt("MYSQL_REALTIME_RETRY_DELAY_MS", 20)) * time.Millisecond,
			})
			mysqlStoreUpdater := realtimeUpdater(storeMetricUpdater{store: "mysql", delegate: mysqlDelegate, registry: registry})
			mysqlUpdater := mysqlStoreUpdater
			minInterval := time.Duration(envInt("MYSQL_REALTIME_MIN_INTERVAL_MS", 10000)) * time.Millisecond
			if redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false" {
				asyncUpdater := newAsyncSecondaryRealtimeUpdater(
					nil,
					mysqlStoreUpdater,
					envInt("MYSQL_REALTIME_ASYNC_QUEUE_SIZE", 20000),
					envInt("MYSQL_REALTIME_ASYNC_WORKERS", 4),
					registry,
					logger,
				)
				closeRealtimeUpdaters = asyncUpdater.Close
				mysqlUpdater = asyncUpdater
			}
			throttleRetention := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_RETENTION_HOURS", 72)) * time.Hour
			throttleCleanupInterval := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second
			registry.SetGauge("vehicle_realtime_throttle_cache_retention_seconds", metrics.Labels{"store": "mysql"}, throttleRetention.Seconds())
			registry.SetGauge("vehicle_realtime_throttle_cache_cleanup_interval_seconds", metrics.Labels{"store": "mysql"}, throttleCleanupInterval.Seconds())
			mysqlUpdater = newThrottledRealtimeUpdaterWithCache(mysqlUpdater, minInterval, "mysql", registry, throttleRetention, throttleCleanupInterval)
			updater = composeRealtimeUpdater(redisUpdater, mysqlUpdater)
			logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot", "plate_binding_table", bindingTable, "plate_cache_ttl_seconds", envInt("PLATE_CACHE_TTL_SECONDS", 600), "plate_cache_max_entries", plateCacheMaxEntries, "async_enabled", redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false", "min_interval_ms", minInterval.Milliseconds(), "throttle_cache_retention_seconds", throttleRetention.Seconds(), "throttle_cache_cleanup_interval_seconds", throttleCleanupInterval.Seconds())
		}
		if apiEnabled {
			metricHandler := stats.NewMetricHandler(stats.NewMetricRepository(db))
			mux.Handle("/api/stats/daily-metrics", metricHandler)
			mux.Handle("/api/stats/daily-metrics/sources", metricHandler)
			mux.Handle("/api/stats/daily-metrics/sources/quality", metricHandler)
			mux.Handle("/api/stats/daily-metrics/sources/selection", metricHandler)
			mux.Handle("/api/stats/daily-metrics/diagnostics", metricHandler)
			mux.Handle("/api/stats/daily-metrics/diagnostics/summary", metricHandler)
			mux.Handle("/api/stats/daily-metrics/diagnostics/reasons", metricHandler)
			mux.Handle("/api/stats/daily-metrics/diagnostics/field-status", metricHandler)
			dataSourceHandler := stats.NewDataSourceHandler(stats.NewDataSourceRepository(db))
			mux.Handle("/api/stats/data-sources", dataSourceHandler)
			mux.Handle("/api/stats/data-sources/", dataSourceHandler)
			mux.Handle("/api/realtime/snapshots", realtime.NewSnapshotQueryHandler(realtime.NewSnapshotQueryRepository(db)))
			mux.Handle("/api/realtime/locations", realtime.NewLocationQueryHandler(realtime.NewLocationQueryRepository(db)))
			logger.Info("stats mysql query enabled")
		}
	} else {
		mysqlUnavailable := func(w http.ResponseWriter, _ *http.Request) {
			w.Header().Set("Content-Type", "application/json")
			w.WriteHeader(http.StatusServiceUnavailable)
			_ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"})
		}
		if apiEnabled {
			mux.HandleFunc("/api/stats/daily-metrics", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/sources", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/sources/quality", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/sources/selection", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/diagnostics", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/diagnostics/summary", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/diagnostics/reasons", mysqlUnavailable)
			mux.HandleFunc("/api/stats/daily-metrics/diagnostics/field-status", mysqlUnavailable)
			mux.HandleFunc("/api/stats/data-sources", mysqlUnavailable)
			mux.HandleFunc("/api/stats/data-sources/", mysqlUnavailable)
			mux.HandleFunc("/api/realtime/snapshots", mysqlUnavailable)
			mux.HandleFunc("/api/realtime/locations", mysqlUnavailable)
			logger.Warn("MYSQL_DSN is empty; stats query api disabled")
		}
	}
	defer closeStats()
	defer closeRealtimeUpdaters()
	var realtimeConsumers sync.WaitGroup
	if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); writerEnabled && len(brokers) > 0 {
		if updater == nil {
			logger.Warn("KAFKA_BROKERS is configured but realtime kafka projector has no enabled store")
		} else {
			kafkaTopics := kafkaTopicsFromEnv()
			workerCount := envInt("REALTIME_WORKERS", 3)
			if err := validateRealtimeConsumerConfig(kafkaTopics, workerCount); err != nil {
				logger.Error("invalid realtime writer config", "error", err)
				os.Exit(1)
			}
			metrics.RegisterKafkaConsumerInfo(registry, "vehicle-realtime-api", env("KAFKA_GROUP", "go-realtime-api"), kafkaTopics)
			registry.SetGauge("vehicle_realtime_config", metrics.Labels{"setting": "workers"}, float64(workerCount))
			for workerID := 1; workerID <= workerCount; workerID++ {
				realtimeConsumers.Add(1)
				go func(id int) {
					defer realtimeConsumers.Done()
					consumeKafkaWorker(
						ctx,
						logger,
						registry,
						updater,
						brokers,
						kafkaTopics,
						envInt("REALTIME_BATCH_SIZE", 100),
						time.Duration(envInt("REALTIME_BATCH_WAIT_MS", 20))*time.Millisecond,
						time.Duration(envInt("REALTIME_FAILURE_RETRY_DELAY_MS", 100))*time.Millisecond,
						id,
					)
				}(workerID)
			}
		}
	} else if writerEnabled {
		logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only")
	} else {
		logger.Info("realtime kafka projector disabled by role", "role", role)
	}
	defer func() {
		stop()
		realtimeConsumers.Wait()
	}()
	closeHistory := func() {}
	if apiEnabled {
		if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" {
			driver := env("TDENGINE_DRIVER", "taosWS")
			db, err := sql.Open(driver, dsn)
			if err != nil {
				logger.Error("history tdengine open failed", "driver", driver, "error", err)
				os.Exit(1)
			}
			if err := db.PingContext(ctx); err != nil {
				_ = db.Close()
				logger.Error("history tdengine ping failed", "driver", driver, "error", err)
				os.Exit(1)
			}
			healthChecks = append(healthChecks, health.Check{Name: "tdengine", Check: db.PingContext})
			closeHistory = func() { _ = db.Close() }
			database := env("TDENGINE_DATABASE", history.DefaultDatabase)
			rawFrameHandler := history.NewRawFrameHandler(history.NewRawFrameRepository(db, database))
			mux.Handle("/api/history/raw-frames", rawFrameHandler)
			mux.Handle("/api/history/raw-frames/query", rawFrameHandler)
			mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database)))
			logger.Info("history query enabled", "driver", driver, "database", database)
		} else {
			historyUnavailable := func(w http.ResponseWriter, _ *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				w.WriteHeader(http.StatusServiceUnavailable)
				_ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"})
			}
			mux.HandleFunc("/api/history/raw-frames", historyUnavailable)
			mux.HandleFunc("/api/history/raw-frames/query", historyUnavailable)
			mux.HandleFunc("/api/history/locations", historyUnavailable)
			logger.Warn("TDENGINE_DSN is empty; history query api disabled")
		}
	} else {
		logger.Info("tdengine dependency disabled by role", "role", role)
	}
	defer closeHistory()
	healthHandler := health.NewHandler(serviceName, healthChecks)
	mux.Handle("/healthz", healthHandler)
	mux.Handle("/readyz", healthHandler)

	server := &http.Server{
		Addr:              env("HTTP_ADDR", ":20210"),
		Handler:           mux,
		ReadHeaderTimeout: 5 * time.Second,
	}
	go func() {
		<-ctx.Done()
		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		_ = server.Shutdown(shutdownCtx)
	}()

	logger.Info("realtime service started", "service", serviceName, "addr", server.Addr, "role", role)
	if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
		logger.Error("http server failed", "error", err)
		os.Exit(1)
	}
}

type realtimeLogger interface {
	Info(string, ...any)
	Error(string, ...any)
	Warn(string, ...any)
}

func consumeKafkaWorker(ctx context.Context, logger realtimeLogger, registry *metrics.Registry, updater realtimeUpdater, brokers []string, kafkaTopics []string, batchSize int, batchWait time.Duration, retryDelay time.Duration, workerID int) {
	reader := kafka.NewReader(realtimeReaderConfig(brokers, kafkaTopics))
	defer reader.Close()
	workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)}
	registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 1)
	defer registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 0)
	logger.Info("realtime kafka consumer started", "worker", workerID, "topics", strings.Join(kafkaTopics, ","), "batch_size", batchSize, "batch_wait_ms", batchWait.Milliseconds())
	for {
		message, err := reader.FetchMessage(ctx)
		if err != nil {
			if ctx.Err() != nil {
				return
			}
			logger.Error("kafka fetch failed", "error", err)
			continue
		}
		batch := collectRealtimeBatch(ctx, reader, message, batchSize, batchWait)
		processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, reader, batch, retryDelay, workerLabels)
	}
}

func validateRealtimeConsumerConfig(kafkaTopics []string, workers int) error {
	if len(kafkaTopics) == 0 {
		return errors.New("KAFKA_TOPICS must include raw topics")
	}
	for _, topic := range kafkaTopics {
		if _, ok := topics.ProtocolForKnownRawTopic(topic); !ok {
			return fmt.Errorf("realtime-writer consumes known raw topics only, got %q", topic)
		}
	}
	if workers <= 0 {
		return errors.New("REALTIME_WORKERS must be greater than zero")
	}
	return nil
}

func realtimeReaderConfig(brokers []string, kafkaTopics []string) kafka.ReaderConfig {
	return kafka.ReaderConfig{
		Brokers:     brokers,
		GroupID:     env("KAFKA_GROUP", "go-realtime-api"),
		GroupTopics: kafkaTopics,
		MinBytes:    1,
		MaxBytes:    10e6,
		StartOffset: kafka.LastOffset,
	}
}

func kafkaTopicsFromEnv() []string {
	defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",")
	return splitCSV(env("KAFKA_TOPICS", defaultTopics))
}

const kafkaMessageOperationTimeout = 30 * time.Second

type realtimeUpdater interface {
	Update(context.Context, envelope.FrameEnvelope) error
}

type realtimeBatchUpdater interface {
	UpdateBatch(context.Context, []envelope.FrameEnvelope) error
}

type fastRealtimeUpdater struct {
	repository *realtime.Repository
}

func (u fastRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	return u.repository.FastUpdate(ctx, env)
}

func (u fastRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	if u.repository == nil || len(envs) == 0 {
		return nil
	}
	return u.repository.FastUpdateBatch(ctx, envs)
}

type storeMetricUpdater struct {
	store    string
	delegate realtimeUpdater
	registry *metrics.Registry
}

var realtimeStoreUpdateDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}

func (u storeMetricUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	if u.delegate == nil {
		return nil
	}
	started := time.Now()
	err := u.delegate.Update(ctx, env)
	elapsed := time.Since(started)
	status := "ok"
	if err != nil {
		status = "error"
	}
	if u.registry != nil {
		labels := metrics.Labels{
			"store":    u.store,
			"protocol": string(env.Protocol),
			"status":   status,
		}
		elapsedMS := float64(elapsed.Milliseconds())
		u.registry.IncCounter("vehicle_realtime_store_updates_total", labels)
		u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS)
		u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS)
		metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels)
	}
	return err
}

func (u storeMetricUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	if u.delegate == nil || len(envs) == 0 {
		return nil
	}
	batchDelegate, ok := u.delegate.(realtimeBatchUpdater)
	if !ok {
		for _, env := range envs {
			if err := u.Update(ctx, env); err != nil {
				return err
			}
		}
		return nil
	}
	started := time.Now()
	err := batchDelegate.UpdateBatch(ctx, envs)
	elapsed := time.Since(started)
	status := statusFromError(err)
	if u.registry != nil {
		elapsedMS := float64(elapsed.Milliseconds())
		counts := realtimeProtocolCounts(envs)
		for protocol, count := range counts {
			labels := metrics.Labels{
				"store":    u.store,
				"protocol": string(protocol),
				"status":   status,
			}
			u.registry.AddCounter("vehicle_realtime_store_updates_total", labels, float64(count))
			u.registry.IncCounter("vehicle_realtime_store_batch_updates_total", labels)
			u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS)
			u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS)
			metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels)
		}
	}
	return err
}

func realtimeProtocolCounts(envs []envelope.FrameEnvelope) map[envelope.Protocol]int {
	counts := make(map[envelope.Protocol]int)
	for _, env := range envs {
		counts[env.Protocol]++
	}
	return counts
}

type retryRealtimeUpdater struct {
	delegate realtimeUpdater
	attempts int
	delay    time.Duration
}

func (u retryRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	if u.delegate == nil {
		return nil
	}
	attempts := u.attempts
	if attempts <= 0 {
		attempts = 1
	}
	var err error
	for attempt := 1; attempt <= attempts; attempt++ {
		err = u.delegate.Update(ctx, env)
		if err == nil || !isTransientMySQLWriteError(err) || attempt == attempts {
			return err
		}
		if u.delay <= 0 {
			continue
		}
		timer := time.NewTimer(u.delay)
		select {
		case <-ctx.Done():
			timer.Stop()
			return ctx.Err()
		case <-timer.C:
		}
	}
	return err
}

func (u retryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	if u.delegate == nil || len(envs) == 0 {
		return nil
	}
	attempts := u.attempts
	if attempts <= 0 {
		attempts = 1
	}
	var err error
	for attempt := 1; attempt <= attempts; attempt++ {
		err = updateRealtimeBatch(ctx, u.delegate, envs)
		if err == nil || !isTransientMySQLWriteError(err) || attempt == attempts {
			return err
		}
		if u.delay <= 0 {
			continue
		}
		timer := time.NewTimer(u.delay)
		select {
		case <-ctx.Done():
			timer.Stop()
			return ctx.Err()
		case <-timer.C:
		}
	}
	return err
}

type throttledRealtimeUpdater struct {
	delegate        realtimeUpdater
	interval        time.Duration
	store           string
	registry        *metrics.Registry
	cacheRetention  time.Duration
	cleanupInterval time.Duration
	mu              sync.Mutex
	lastMS          map[string]int64
	lastCleanupMS   int64
	lastDeleted     int
}

func newThrottledRealtimeUpdater(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry) realtimeUpdater {
	return newThrottledRealtimeUpdaterWithCache(delegate, interval, store, registry, defaultThrottleCacheRetention, defaultThrottleCacheCleanupInterval)
}

func newThrottledRealtimeUpdaterWithCache(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry, cacheRetention time.Duration, cleanupInterval time.Duration) realtimeUpdater {
	if delegate == nil || interval <= 0 {
		return delegate
	}
	if cacheRetention < 0 {
		cacheRetention = 0
	}
	if cleanupInterval < 0 {
		cleanupInterval = 0
	}
	return &throttledRealtimeUpdater{
		delegate:        delegate,
		interval:        interval,
		store:           store,
		registry:        registry,
		cacheRetention:  cacheRetention,
		cleanupInterval: cleanupInterval,
		lastMS:          map[string]int64{},
	}
}

func (u *throttledRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	if u.delegate == nil {
		return nil
	}
	pass, status := u.shouldPass(env)
	u.record(status, env)
	u.recordCacheMetrics()
	if !pass {
		return nil
	}
	return u.delegate.Update(ctx, env)
}

func (u *throttledRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	for _, env := range envs {
		if err := u.Update(ctx, env); err != nil {
			return err
		}
	}
	return nil
}

func (u *throttledRealtimeUpdater) shouldPass(env envelope.FrameEnvelope) (bool, string) {
	if !envelope.IsRealtimeTelemetryFrame(env) {
		return false, "skipped_non_realtime"
	}
	vin := strings.TrimSpace(env.VIN)
	if vin == "" {
		return false, "skipped_missing_vin"
	}
	eventMS, ok := envelope.NormalizedEventTimeMS(env)
	if !ok || eventMS <= 0 {
		eventMS = env.ReceivedAtMS
	}
	if eventMS <= 0 {
		eventMS = time.Now().UnixMilli()
	}
	key := string(env.Protocol) + "\x00" + vin
	intervalMS := u.interval.Milliseconds()
	u.mu.Lock()
	defer u.mu.Unlock()
	if u.lastMS == nil {
		u.lastMS = map[string]int64{}
	}
	u.cleanupLocked(throttleCleanupTimeMS(env, eventMS))
	if last := u.lastMS[key]; last > 0 && eventMS < last+intervalMS {
		return false, "skipped_interval"
	}
	u.lastMS[key] = eventMS
	return true, "passed"
}

func (u *throttledRealtimeUpdater) cleanupLocked(nowMS int64) {
	if u.cacheRetention <= 0 || nowMS <= 0 {
		return
	}
	intervalMS := u.cleanupInterval.Milliseconds()
	if intervalMS > 0 && u.lastCleanupMS > 0 && nowMS < u.lastCleanupMS+intervalMS {
		return
	}
	u.lastCleanupMS = nowMS
	cutoff := nowMS - u.cacheRetention.Milliseconds()
	deleted := 0
	for key, last := range u.lastMS {
		if last > 0 && last < cutoff {
			delete(u.lastMS, key)
			deleted++
		}
	}
	u.lastDeleted = deleted
}

func throttleCleanupTimeMS(env envelope.FrameEnvelope, fallbackMS int64) int64 {
	if env.ReceivedAtMS > 0 {
		return env.ReceivedAtMS
	}
	if fallbackMS > 0 {
		return fallbackMS
	}
	return time.Now().UnixMilli()
}

func (u *throttledRealtimeUpdater) record(status string, env envelope.FrameEnvelope) {
	if u.registry == nil {
		return
	}
	u.registry.IncCounter("vehicle_realtime_throttle_total", metrics.Labels{
		"store":    u.store,
		"protocol": string(env.Protocol),
		"status":   status,
	})
}

func (u *throttledRealtimeUpdater) recordCacheMetrics() {
	if u.registry == nil {
		return
	}
	u.mu.Lock()
	entries := len(u.lastMS)
	lastDeleted := u.lastDeleted
	lastCleanupMS := u.lastCleanupMS
	u.mu.Unlock()
	labels := metrics.Labels{"store": u.store}
	u.registry.SetGauge("vehicle_realtime_throttle_cache_entries", labels, float64(entries))
	u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_deleted", labels, float64(lastDeleted))
	if lastCleanupMS > 0 {
		u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_unix_seconds", labels, float64(lastCleanupMS)/1000)
	}
}

func isTransientMySQLWriteError(err error) bool {
	if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
		return false
	}
	text := strings.ToLower(strings.TrimSpace(err.Error()))
	return strings.Contains(text, "deadlock") ||
		strings.Contains(text, "error 1213") ||
		strings.Contains(text, "40001") ||
		strings.Contains(text, "lock wait timeout") ||
		strings.Contains(text, "error 1205") ||
		strings.Contains(text, "timeout") ||
		strings.Contains(text, "temporary") ||
		strings.Contains(text, "temporarily") ||
		strings.Contains(text, "connection refused") ||
		strings.Contains(text, "connection reset") ||
		strings.Contains(text, "connection closed") ||
		strings.Contains(text, "broken pipe") ||
		strings.Contains(text, "bad connection") ||
		strings.Contains(text, "invalid connection") ||
		strings.Contains(text, "i/o timeout") ||
		text == "eof" ||
		strings.Contains(text, "unexpected eof") ||
		strings.Contains(text, "server is down") ||
		strings.Contains(text, "network is unreachable") ||
		strings.Contains(text, "no route to host")
}

type compositeRealtimeUpdater struct {
	primary   realtimeUpdater
	secondary realtimeUpdater
}

func composeRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater) realtimeUpdater {
	if primary == nil {
		return secondary
	}
	if secondary == nil {
		return primary
	}
	return compositeRealtimeUpdater{primary: primary, secondary: secondary}
}

func (u compositeRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	if u.primary != nil {
		if err := u.primary.Update(ctx, env); err != nil {
			return err
		}
	}
	if u.secondary == nil {
		return nil
	}
	return u.secondary.Update(ctx, env)
}

func (u compositeRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	if u.primary != nil {
		if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil {
			return err
		}
	}
	return updateRealtimeBatch(ctx, u.secondary, envs)
}

type asyncSecondaryRealtimeUpdater struct {
	primary   realtimeUpdater
	secondary realtimeUpdater
	queue     chan envelope.FrameEnvelope
	registry  *metrics.Registry
	logger    realtimeLogger
	closeOnce sync.Once
	closed    chan struct{}
	wg        sync.WaitGroup
}

func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater, queueSize int, workers int, registry *metrics.Registry, logger realtimeLogger) *asyncSecondaryRealtimeUpdater {
	if queueSize <= 0 {
		queueSize = 20000
	}
	if workers <= 0 {
		workers = 4
	}
	updater := &asyncSecondaryRealtimeUpdater{
		primary:   primary,
		secondary: secondary,
		queue:     make(chan envelope.FrameEnvelope, queueSize),
		registry:  registry,
		logger:    logger,
		closed:    make(chan struct{}),
	}
	updater.recordQueueCapacity()
	updater.wg.Add(workers)
	for i := 0; i < workers; i++ {
		go updater.run()
	}
	return updater
}

func (u *asyncSecondaryRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
	if u.primary != nil {
		if err := u.primary.Update(ctx, env); err != nil {
			return err
		}
	}
	if u.secondary == nil {
		return nil
	}
	select {
	case <-u.closed:
		u.recordQueueMetric(env, "closed")
		u.recordQueueDepth(env)
		return nil
	default:
	}
	select {
	case u.queue <- env:
		u.recordQueueMetric(env, "queued")
		u.recordQueueDepth(env)
		return nil
	case <-u.closed:
		u.recordQueueMetric(env, "closed")
		u.recordQueueDepth(env)
		return nil
	default:
		u.recordQueueMetric(env, "dropped")
		u.recordQueueDepth(env)
		return nil
	}
}

func (u *asyncSecondaryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
	if u.primary != nil {
		if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil {
			return err
		}
	}
	if u.secondary == nil {
		return nil
	}
	for _, env := range envs {
		if err := u.enqueueSecondary(env); err != nil {
			return err
		}
	}
	return nil
}

func (u *asyncSecondaryRealtimeUpdater) enqueueSecondary(env envelope.FrameEnvelope) error {
	select {
	case <-u.closed:
		u.recordQueueMetric(env, "closed")
		u.recordQueueDepth(env)
		return nil
	default:
	}
	select {
	case u.queue <- env:
		u.recordQueueMetric(env, "queued")
		u.recordQueueDepth(env)
		return nil
	case <-u.closed:
		u.recordQueueMetric(env, "closed")
		u.recordQueueDepth(env)
		return nil
	default:
		u.recordQueueMetric(env, "dropped")
		u.recordQueueDepth(env)
		return nil
	}
}

func (u *asyncSecondaryRealtimeUpdater) run() {
	defer u.wg.Done()
	for {
		select {
		case env := <-u.queue:
			u.updateSecondary(env)
		case <-u.closed:
			for {
				select {
				case env := <-u.queue:
					u.updateSecondary(env)
				default:
					return
				}
			}
		}
	}
}

func (u *asyncSecondaryRealtimeUpdater) updateSecondary(env envelope.FrameEnvelope) {
	u.recordQueueDepth(env)
	secondaryCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout)
	if err := u.secondary.Update(secondaryCtx, env); err != nil && u.logger != nil {
		u.logger.Error("async realtime secondary update failed", "protocol", env.Protocol, "vin", env.VIN, "event_id", env.StableEventID(), "error", err)
	}
	cancel()
	u.recordQueueDepth(env)
}

func (u *asyncSecondaryRealtimeUpdater) recordQueueMetric(env envelope.FrameEnvelope, status string) {
	if u.registry == nil {
		return
	}
	u.registry.IncCounter("vehicle_realtime_async_queue_total", metrics.Labels{
		"store":    "mysql",
		"protocol": string(env.Protocol),
		"status":   status,
	})
}

func (u *asyncSecondaryRealtimeUpdater) recordQueueDepth(env envelope.FrameEnvelope) {
	if u.registry == nil {
		return
	}
	u.registry.SetGauge("vehicle_realtime_async_queue_depth", metrics.Labels{
		"store":    "mysql",
		"protocol": string(env.Protocol),
	}, float64(len(u.queue)))
}

func (u *asyncSecondaryRealtimeUpdater) recordQueueCapacity() {
	if u.registry == nil {
		return
	}
	u.registry.SetGauge("vehicle_realtime_async_queue_capacity", metrics.Labels{
		"store": "mysql",
	}, float64(cap(u.queue)))
}

func (u *asyncSecondaryRealtimeUpdater) Close() {
	u.closeOnce.Do(func() {
		close(u.closed)
	})
	u.wg.Wait()
}

type kafkaMessageCommitter interface {
	CommitMessages(context.Context, ...kafka.Message) error
}

type kafkaMessageFetcher interface {
	FetchMessage(context.Context) (kafka.Message, error)
}

type realtimeBatchItem struct {
	message   kafka.Message
	env       envelope.FrameEnvelope
	valid     bool
	processed bool
}

type realtimeBatchOutcome struct {
	commitMessages []kafka.Message
	retryMessages  []kafka.Message
}

func collectRealtimeBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
	if maxSize <= 1 {
		return []kafka.Message{first}
	}
	if maxWait <= 0 {
		maxWait = 20 * time.Millisecond
	}
	batch := []kafka.Message{first}
	deadline := time.Now().Add(maxWait)
	for len(batch) < maxSize {
		remaining := time.Until(deadline)
		if remaining <= 0 {
			break
		}
		fetchCtx, cancel := context.WithTimeout(ctx, remaining)
		message, err := fetcher.FetchMessage(fetchCtx)
		cancel()
		if err != nil {
			break
		}
		batch = append(batch, message)
	}
	return batch
}

func processRealtimeBatch(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message) realtimeBatchOutcome {
	return processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, messages, nil)
}

func processRealtimeBatchForWorker(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) realtimeBatchOutcome {
	if len(messages) == 0 {
		return realtimeBatchOutcome{}
	}
	messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
	defer cancel()

	setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), 0)
	defer setRealtimeBatchPendingForWorker(registry, workerLabels, 0, 0)
	items := make([]realtimeBatchItem, 0, len(messages))
	envs := make([]envelope.FrameEnvelope, 0, len(messages))
	for _, message := range messages {
		itemIndex := len(items)
		items = append(items, realtimeBatchItem{message: message})
		addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "received")
		addRealtimeLagMetric(registry, message)
		var env envelope.FrameEnvelope
		if err := json.Unmarshal(message.Value, &env); err != nil {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json")
			logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
			items[itemIndex].processed = true
			continue
		}
		if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status)
			logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err)
			items[itemIndex].processed = true
			continue
		}
		if status := realtimeProjectionSkipStatus(env); status != "" {
			addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status)
			items[itemIndex].processed = true
			continue
		}
		items[itemIndex].env = env
		items[itemIndex].valid = true
		envs = append(envs, env)
	}
	setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), len(envs))
	if len(envs) > 0 {
		started := time.Now()
		err := updateRealtimeBatch(messageCtx, updater, envs)
		recordRealtimeBatchFlush(registry, statusFromError(err), len(messages), len(envs), time.Since(started))
		if err != nil {
			logger.Error("realtime batch update failed; falling back to per-message isolation", "messages", len(messages), "valid_messages", len(envs), "error", err)
			if !fallbackRealtimeBatch(messageCtx, logger, registry, updater, items) {
				committed, commitErr := commitRealtimeProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items)
				return realtimeFailureOutcome(messages, committed, commitErr)
			}
		} else {
			for index := range items {
				if items[index].valid {
					items[index].processed = true
					addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok")
				}
			}
		}
	}
	if err := committer.CommitMessages(messageCtx, messages...); err != nil {
		for _, message := range messages {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
		}
		first := messages[0]
		logger.Error("kafka batch commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err)
		return realtimeBatchOutcome{commitMessages: messages}
	}
	for _, message := range messages {
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
	}
	return realtimeBatchOutcome{}
}

func realtimeFailureOutcome(messages []kafka.Message, committed []kafka.Message, commitErr error) realtimeBatchOutcome {
	outcome := realtimeBatchOutcome{
		retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed),
	}
	if commitErr != nil {
		outcome.commitMessages = committed
	}
	return outcome
}

func processRealtimeBatchReliably(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) {
	processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, committer, messages, retryDelay, nil)
}

func processRealtimeBatchReliablyForWorker(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
	defer registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, 0)
	pending := messages
	for len(pending) > 0 {
		outcome := processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, pending, workerLabels)
		if len(outcome.commitMessages) > 0 {
			registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"})
			if !retryRealtimeCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) {
				return
			}
		}
		pending = outcome.retryMessages
		registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, float64(len(pending)))
		if len(pending) == 0 {
			return
		}
		registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "write_error"})
		if !waitForRealtimeRetry(ctx, retryDelay) {
			return
		}
	}
}

func retryRealtimeCommit(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool {
	for len(messages) > 0 {
		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
		err := committer.CommitMessages(operationCtx, messages...)
		cancel()
		if err == nil {
			for _, message := range messages {
				addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
			}
			return true
		}
		for _, message := range messages {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
		}
		first := messages[0]
		logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err)
		registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"})
		if !waitForRealtimeRetry(ctx, retryDelay) {
			return false
		}
	}
	return true
}

func waitForRealtimeRetry(ctx context.Context, retryDelay time.Duration) bool {
	if retryDelay <= 0 {
		retryDelay = 100 * time.Millisecond
	}
	timer := time.NewTimer(retryDelay)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return false
	case <-timer.C:
		return true
	}
}

func fallbackRealtimeBatch(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, items []realtimeBatchItem) bool {
	for index := range items {
		if !items[index].valid {
			continue
		}
		if err := updater.Update(ctx, items[index].env); err != nil {
			message := items[index].message
			addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error")
			logger.Error("realtime fallback update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", items[index].env.StableEventID(), "error", err)
			return false
		}
		items[index].processed = true
		addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok")
	}
	return true
}

func commitRealtimeProcessedPrefixAfterFailure(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, committer kafkaMessageCommitter, items []realtimeBatchItem) ([]kafka.Message, error) {
	committable := realtimeProcessedPrefixMessagesByPartition(items)
	if len(committable) == 0 {
		return nil, nil
	}
	if err := committer.CommitMessages(ctx, committable...); err != nil {
		for _, message := range committable {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
		}
		first := committable[0]
		logger.Error("kafka realtime processed-prefix commit failed after batch update failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err)
		return committable, err
	}
	for _, message := range committable {
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
	}
	return committable, nil
}

func realtimeProcessedPrefixMessagesByPartition(items []realtimeBatchItem) []kafka.Message {
	type partitionKey struct {
		topic     string
		partition int
	}
	groups := map[partitionKey][]realtimeBatchItem{}
	for _, item := range items {
		key := partitionKey{topic: item.message.Topic, partition: item.message.Partition}
		groups[key] = append(groups[key], item)
	}
	var keys []partitionKey
	for key := range groups {
		keys = append(keys, key)
	}
	sort.Slice(keys, func(i, j int) bool {
		if keys[i].topic != keys[j].topic {
			return keys[i].topic < keys[j].topic
		}
		return keys[i].partition < keys[j].partition
	})
	var out []kafka.Message
	for _, key := range keys {
		group := groups[key]
		sort.Slice(group, func(i, j int) bool {
			return group[i].message.Offset < group[j].message.Offset
		})
		var previousOffset int64
		for index, item := range group {
			if index > 0 && item.message.Offset != previousOffset+1 {
				break
			}
			if !item.processed {
				break
			}
			out = append(out, item.message)
			previousOffset = item.message.Offset
		}
	}
	return out
}

func updateRealtimeBatch(ctx context.Context, updater realtimeUpdater, envs []envelope.FrameEnvelope) error {
	if updater == nil || len(envs) == 0 {
		return nil
	}
	if batchUpdater, ok := updater.(realtimeBatchUpdater); ok {
		return batchUpdater.UpdateBatch(ctx, envs)
	}
	for _, env := range envs {
		if err := updater.Update(ctx, env); err != nil {
			return err
		}
	}
	return nil
}

func processRealtimeMessage(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, message kafka.Message) {
	messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
	defer cancel()

	addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "received")
	addRealtimeLagMetric(registry, message)
	var env envelope.FrameEnvelope
	if err := json.Unmarshal(message.Value, &env); err != nil {
		addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json")
		logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
		if err := committer.CommitMessages(messageCtx, message); err != nil {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
			logger.Error("kafka commit invalid envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
			return
		}
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
		return
	}
	if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil {
		addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status)
		logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err)
		if err := committer.CommitMessages(messageCtx, message); err != nil {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
			logger.Error("kafka commit mismatched realtime raw envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err)
			return
		}
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
		return
	}
	if status := realtimeProjectionSkipStatus(env); status != "" {
		addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status)
		if err := committer.CommitMessages(messageCtx, message); err != nil {
			addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
			logger.Error("kafka commit skipped realtime envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err)
			return
		}
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
		return
	}
	if err := updater.Update(messageCtx, env); err != nil {
		addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error")
		logger.Error("realtime projector update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
		return
	}
	addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "ok")
	if err := committer.CommitMessages(messageCtx, message); err != nil {
		addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error")
		logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
		return
	}
	addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok")
}

func realtimeProjectionSkipStatus(env envelope.FrameEnvelope) string {
	if !envelope.IsRealtimeTelemetryFrame(env) {
		return "skipped_non_realtime"
	}
	if strings.TrimSpace(env.VIN) == "" {
		return "skipped_missing_vin"
	}
	vehicleKey := strings.TrimSpace(env.VehicleKey())
	if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
		return "skipped_missing_vehicle_key"
	}
	if len(env.ParsedFields) == 0 {
		return "skipped_missing_fields"
	}
	return ""
}

func addRealtimeMetric(registry *metrics.Registry, name string, message kafka.Message, status string) {
	if registry == nil {
		return
	}
	labels := metrics.Labels{
		"topic":  message.Topic,
		"status": status,
	}
	registry.IncCounter(name, labels)
	switch name {
	case "vehicle_realtime_kafka_messages_total":
		metrics.RecordLastActivity(registry, "vehicle_realtime_last_message_unix_seconds", labels)
	case "vehicle_realtime_updates_total":
		metrics.RecordLastActivity(registry, "vehicle_realtime_last_update_unix_seconds", labels)
	case "vehicle_realtime_kafka_commits_total":
		metrics.RecordLastActivity(registry, "vehicle_realtime_last_commit_unix_seconds", labels)
	}
}

func recordPlateCacheStats(registry *metrics.Registry, stats realtime.PlateCacheStats) {
	if registry == nil {
		return
	}
	registry.SetGauge("vehicle_realtime_plate_cache_entries", nil, float64(stats.Entries))
	registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(stats.MaxEntries))
	registry.SetGauge("vehicle_realtime_plate_cache_evictions_total", nil, float64(stats.Evictions))
}

func addRealtimeLagMetric(registry *metrics.Registry, message kafka.Message) {
	if registry == nil {
		return
	}
	registry.SetKafkaLag("vehicle_realtime_kafka_lag", message.Topic, message.Partition, message.Offset, message.HighWaterMark)
}

var realtimeBatchFlushDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}

func recordRealtimeBatchFlush(registry *metrics.Registry, status string, messages int, validMessages int, elapsed time.Duration) {
	if registry == nil {
		return
	}
	labels := metrics.Labels{"status": status}
	elapsedMS := float64(elapsed.Milliseconds())
	registry.IncCounter("vehicle_realtime_batch_flush_total", labels)
	registry.AddCounter("vehicle_realtime_batch_messages_total", labels, float64(messages))
	registry.AddCounter("vehicle_realtime_batch_valid_messages_total", labels, float64(validMessages))
	registry.SetGauge("vehicle_realtime_batch_flush_duration_ms", labels, elapsedMS)
	registry.ObserveHistogram("vehicle_realtime_batch_flush_duration_ms_histogram", labels, realtimeBatchFlushDurationBucketsMS, elapsedMS)
}

func setRealtimeBatchPending(registry *metrics.Registry, messages int, validMessages int) {
	setRealtimeBatchPendingForWorker(registry, nil, messages, validMessages)
}

func setRealtimeBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int, validMessages int) {
	if registry == nil {
		return
	}
	registry.SetGauge("vehicle_realtime_batch_pending_messages", workerLabels, float64(messages))
	registry.SetGauge("vehicle_realtime_batch_pending_valid_messages", workerLabels, float64(validMessages))
}

func statusFromError(err error) string {
	if err != nil {
		return "error"
	}
	return "ok"
}

func realtimeRoleFromEnv() string {
	return normalizeRealtimeRole(env("REALTIME_ROLE", "combined"))
}

func normalizeRealtimeRole(value string) string {
	switch strings.ToLower(strings.TrimSpace(value)) {
	case "api", "query", "read", "readonly", "read-only":
		return "api"
	case "writer", "projector", "consumer", "write":
		return "writer"
	case "combined", "all", "both", "":
		return "combined"
	default:
		return "combined"
	}
}

func realtimeRoleAllowsAPI(role string) bool {
	switch normalizeRealtimeRole(role) {
	case "writer":
		return false
	default:
		return true
	}
}

func realtimeRoleAllowsWriter(role string) bool {
	switch normalizeRealtimeRole(role) {
	case "api":
		return false
	default:
		return true
	}
}

func realtimeRoleNeedsRedis(apiEnabled bool, writerEnabled bool, redisProjectorEnabled bool) bool {
	return apiEnabled || (writerEnabled && redisProjectorEnabled)
}

func realtimeServiceName(role string) string {
	if normalizeRealtimeRole(role) == "writer" {
		return "vehicle-realtime-writer"
	}
	return "vehicle-realtime-api"
}

func env(key string, fallback string) string {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	return value
}

func envInt(key string, fallback int) int {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	parsed, err := strconv.Atoi(value)
	if err != nil {
		return fallback
	}
	return parsed
}

func boolGauge(value bool) float64 {
	if value {
		return 1
	}
	return 0
}

func splitCSV(value string) []string {
	var out []string
	for _, item := range strings.Split(value, ",") {
		item = strings.TrimSpace(item)
		if item != "" {
			out = append(out, item)
		}
	}
	return out
}