package main import ( "context" "database/sql" "encoding/json" "errors" "fmt" "net/http" "os" "os/signal" "sort" "strconv" "strings" "sync" "syscall" "time" _ "github.com/go-sql-driver/mysql" "github.com/redis/go-redis/v9" "github.com/segmentio/kafka-go" _ "github.com/taosdata/driver-go/v3/taosWS" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/health" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/history" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics" ) const ( defaultThrottleCacheRetention = 72 * time.Hour defaultThrottleCacheCleanupInterval = 10 * time.Minute ) func main() { role := realtimeRoleFromEnv() apiEnabled := realtimeRoleAllowsAPI(role) writerEnabled := realtimeRoleAllowsWriter(role) redisProjectorEnabled := env("REALTIME_REDIS_PROJECTOR_ENABLED", "true") != "false" serviceName := realtimeServiceName(role) logger := observability.NewLogger(serviceName) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() mux := http.NewServeMux() registry := metrics.NewRegistry() metrics.RegisterServiceInfo(registry, serviceName) registry.SetGauge("vehicle_realtime_role_info", metrics.Labels{"role": role}, 1) registry.SetGauge("vehicle_realtime_writer_enabled", nil, boolGauge(writerEnabled)) registry.SetGauge("vehicle_realtime_redis_projector_enabled", nil, boolGauge(redisProjectorEnabled)) healthChecks := []health.Check{} var client *redis.Client var repository *realtime.Repository if realtimeRoleNeedsRedis(apiEnabled, writerEnabled, redisProjectorEnabled) { client = redis.NewClient(&redis.Options{ Addr: env("REDIS_ADDR", "127.0.0.1:6379"), Username: env("REDIS_USERNAME", ""), Password: env("REDIS_PASSWORD", ""), DB: envInt("REDIS_DB", 0), }) defer client.Close() if err := client.Ping(ctx).Err(); err != nil { logger.Error("redis ping failed", "error", err) os.Exit(1) } repository = realtime.NewRepository(client, realtime.Config{ OnlineTTL: time.Duration(envInt("ONLINE_TTL_SECONDS", envInt("REALTIME_ONLINE_TTL_SECONDS", 60))) * time.Second, }) healthChecks = append(healthChecks, health.Check{Name: "redis", Check: func(ctx context.Context) error { return client.Ping(ctx).Err() }}) } else { logger.Info("redis dependency disabled by role", "role", role, "redis_projector_enabled", redisProjectorEnabled) } mux.Handle("/metrics", metrics.NewHandler(registry)) if apiEnabled { mux.Handle("/openapi.json", realtime.NewOpenAPIHandler()) mux.Handle("/swagger-ui/", realtime.NewSwaggerUIHandler()) mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository)) mux.Handle("/api/realtime/online", realtime.NewOnlineIndexHandler(repository)) mux.Handle("/api/debug/pipeline", realtime.NewPipelineDebugHandler(repository)) } else { logger.Info("realtime query api disabled by role", "role", role) } closeStats := func() {} closeRealtimeUpdaters := func() {} var redisUpdater realtimeUpdater if writerEnabled && redisProjectorEnabled { redisUpdater = storeMetricUpdater{ store: "redis", delegate: fastRealtimeUpdater{repository: repository}, registry: registry, } } else { logger.Info("realtime redis kafka projector disabled; redis query api still enabled") } var updater realtimeUpdater = redisUpdater if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" { db, err := sql.Open("mysql", dsn) if err != nil { logger.Error("stats mysql open failed", "error", err) os.Exit(1) } if err := db.PingContext(ctx); err != nil { _ = db.Close() logger.Error("stats mysql ping failed", "error", err) os.Exit(1) } healthChecks = append(healthChecks, health.Check{Name: "mysql", Check: db.PingContext}) closeStats = func() { _ = db.Close() } bindingTable := env("VEHICLE_IDENTITY_TABLE", "vehicle_identity_binding") plateCacheMaxEntries := envInt("PLATE_CACHE_MAX_ENTRIES", 200000) plateResolver := realtime.NewCachedPlateResolverWithMaxEntries( realtime.NewBindingPlateResolver(db, bindingTable), time.Duration(envInt("PLATE_CACHE_TTL_SECONDS", 600))*time.Second, plateCacheMaxEntries, ) plateResolver.SetStatsObserver(func(stats realtime.PlateCacheStats) { recordPlateCacheStats(registry, stats) }) registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(plateCacheMaxEntries)) snapshotWriter := realtime.NewSnapshotWriterWithPlateResolver(db, plateResolver) if writerEnabled && env("MYSQL_REALTIME_SNAPSHOT_ENABLED", "true") != "false" { if err := snapshotWriter.EnsureSchema(ctx); err != nil { _ = db.Close() logger.Error("realtime snapshot mysql schema bootstrap failed", "error", err) os.Exit(1) } mysqlDelegate := realtimeUpdater(retryRealtimeUpdater{ delegate: snapshotWriter, attempts: envInt("MYSQL_REALTIME_RETRY_ATTEMPTS", 3), delay: time.Duration(envInt("MYSQL_REALTIME_RETRY_DELAY_MS", 20)) * time.Millisecond, }) mysqlStoreUpdater := realtimeUpdater(storeMetricUpdater{store: "mysql", delegate: mysqlDelegate, registry: registry}) mysqlUpdater := mysqlStoreUpdater minInterval := time.Duration(envInt("MYSQL_REALTIME_MIN_INTERVAL_MS", 10000)) * time.Millisecond if redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false" { asyncUpdater := newAsyncSecondaryRealtimeUpdater( nil, mysqlStoreUpdater, envInt("MYSQL_REALTIME_ASYNC_QUEUE_SIZE", 20000), envInt("MYSQL_REALTIME_ASYNC_WORKERS", 4), registry, logger, ) closeRealtimeUpdaters = asyncUpdater.Close mysqlUpdater = asyncUpdater } throttleRetention := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_RETENTION_HOURS", 72)) * time.Hour throttleCleanupInterval := time.Duration(envInt("MYSQL_REALTIME_THROTTLE_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second registry.SetGauge("vehicle_realtime_throttle_cache_retention_seconds", metrics.Labels{"store": "mysql"}, throttleRetention.Seconds()) registry.SetGauge("vehicle_realtime_throttle_cache_cleanup_interval_seconds", metrics.Labels{"store": "mysql"}, throttleCleanupInterval.Seconds()) mysqlUpdater = newThrottledRealtimeUpdaterWithCache(mysqlUpdater, minInterval, "mysql", registry, throttleRetention, throttleCleanupInterval) updater = composeRealtimeUpdater(redisUpdater, mysqlUpdater) logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot", "plate_binding_table", bindingTable, "plate_cache_ttl_seconds", envInt("PLATE_CACHE_TTL_SECONDS", 600), "plate_cache_max_entries", plateCacheMaxEntries, "async_enabled", redisProjectorEnabled && env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false", "min_interval_ms", minInterval.Milliseconds(), "throttle_cache_retention_seconds", throttleRetention.Seconds(), "throttle_cache_cleanup_interval_seconds", throttleCleanupInterval.Seconds()) } if apiEnabled { metricHandler := stats.NewMetricHandler(stats.NewMetricRepository(db)) mux.Handle("/api/stats/daily-metrics", metricHandler) mux.Handle("/api/stats/daily-metrics/sources", metricHandler) mux.Handle("/api/stats/daily-metrics/sources/quality", metricHandler) mux.Handle("/api/stats/daily-metrics/sources/selection", metricHandler) mux.Handle("/api/stats/daily-metrics/diagnostics", metricHandler) mux.Handle("/api/stats/daily-metrics/diagnostics/summary", metricHandler) mux.Handle("/api/stats/daily-metrics/diagnostics/reasons", metricHandler) mux.Handle("/api/stats/daily-metrics/diagnostics/field-status", metricHandler) dataSourceHandler := stats.NewDataSourceHandler(stats.NewDataSourceRepository(db)) mux.Handle("/api/stats/data-sources", dataSourceHandler) mux.Handle("/api/stats/data-sources/", dataSourceHandler) mux.Handle("/api/realtime/snapshots", realtime.NewSnapshotQueryHandler(realtime.NewSnapshotQueryRepository(db))) mux.Handle("/api/realtime/locations", realtime.NewLocationQueryHandler(realtime.NewLocationQueryRepository(db))) logger.Info("stats mysql query enabled") } } else { mysqlUnavailable := func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusServiceUnavailable) _ = json.NewEncoder(w).Encode(map[string]any{"error": "MYSQL_DSN is not configured"}) } if apiEnabled { mux.HandleFunc("/api/stats/daily-metrics", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/sources", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/sources/quality", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/sources/selection", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/diagnostics", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/diagnostics/summary", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/diagnostics/reasons", mysqlUnavailable) mux.HandleFunc("/api/stats/daily-metrics/diagnostics/field-status", mysqlUnavailable) mux.HandleFunc("/api/stats/data-sources", mysqlUnavailable) mux.HandleFunc("/api/stats/data-sources/", mysqlUnavailable) mux.HandleFunc("/api/realtime/snapshots", mysqlUnavailable) mux.HandleFunc("/api/realtime/locations", mysqlUnavailable) logger.Warn("MYSQL_DSN is empty; stats query api disabled") } } defer closeStats() defer closeRealtimeUpdaters() var realtimeConsumers sync.WaitGroup if brokers := splitCSV(os.Getenv("KAFKA_BROKERS")); writerEnabled && len(brokers) > 0 { if updater == nil { logger.Warn("KAFKA_BROKERS is configured but realtime kafka projector has no enabled store") } else { kafkaTopics := kafkaTopicsFromEnv() workerCount := envInt("REALTIME_WORKERS", 3) if err := validateRealtimeConsumerConfig(kafkaTopics, workerCount); err != nil { logger.Error("invalid realtime writer config", "error", err) os.Exit(1) } metrics.RegisterKafkaConsumerInfo(registry, "vehicle-realtime-api", env("KAFKA_GROUP", "go-realtime-api"), kafkaTopics) registry.SetGauge("vehicle_realtime_config", metrics.Labels{"setting": "workers"}, float64(workerCount)) for workerID := 1; workerID <= workerCount; workerID++ { realtimeConsumers.Add(1) go func(id int) { defer realtimeConsumers.Done() consumeKafkaWorker( ctx, logger, registry, updater, brokers, kafkaTopics, envInt("REALTIME_BATCH_SIZE", 100), time.Duration(envInt("REALTIME_BATCH_WAIT_MS", 20))*time.Millisecond, time.Duration(envInt("REALTIME_FAILURE_RETRY_DELAY_MS", 100))*time.Millisecond, id, ) }(workerID) } } } else if writerEnabled { logger.Warn("KAFKA_BROKERS is empty; realtime api will serve existing redis data only") } else { logger.Info("realtime kafka projector disabled by role", "role", role) } defer func() { stop() realtimeConsumers.Wait() }() closeHistory := func() {} if apiEnabled { if dsn := strings.TrimSpace(os.Getenv("TDENGINE_DSN")); dsn != "" { driver := env("TDENGINE_DRIVER", "taosWS") db, err := sql.Open(driver, dsn) if err != nil { logger.Error("history tdengine open failed", "driver", driver, "error", err) os.Exit(1) } if err := db.PingContext(ctx); err != nil { _ = db.Close() logger.Error("history tdengine ping failed", "driver", driver, "error", err) os.Exit(1) } healthChecks = append(healthChecks, health.Check{Name: "tdengine", Check: db.PingContext}) closeHistory = func() { _ = db.Close() } database := env("TDENGINE_DATABASE", history.DefaultDatabase) rawFrameHandler := history.NewRawFrameHandler(history.NewRawFrameRepository(db, database)) mux.Handle("/api/history/raw-frames", rawFrameHandler) mux.Handle("/api/history/raw-frames/query", rawFrameHandler) mux.Handle("/api/history/locations", history.NewLocationHandler(history.NewLocationRepository(db, database))) logger.Info("history query enabled", "driver", driver, "database", database) } else { historyUnavailable := func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusServiceUnavailable) _ = json.NewEncoder(w).Encode(map[string]any{"error": "TDENGINE_DSN is not configured"}) } mux.HandleFunc("/api/history/raw-frames", historyUnavailable) mux.HandleFunc("/api/history/raw-frames/query", historyUnavailable) mux.HandleFunc("/api/history/locations", historyUnavailable) logger.Warn("TDENGINE_DSN is empty; history query api disabled") } } else { logger.Info("tdengine dependency disabled by role", "role", role) } defer closeHistory() healthHandler := health.NewHandler(serviceName, healthChecks) mux.Handle("/healthz", healthHandler) mux.Handle("/readyz", healthHandler) server := &http.Server{ Addr: env("HTTP_ADDR", ":20210"), Handler: mux, ReadHeaderTimeout: 5 * time.Second, } go func() { <-ctx.Done() shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() _ = server.Shutdown(shutdownCtx) }() logger.Info("realtime service started", "service", serviceName, "addr", server.Addr, "role", role) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { logger.Error("http server failed", "error", err) os.Exit(1) } } type realtimeLogger interface { Info(string, ...any) Error(string, ...any) Warn(string, ...any) } func consumeKafkaWorker(ctx context.Context, logger realtimeLogger, registry *metrics.Registry, updater realtimeUpdater, brokers []string, kafkaTopics []string, batchSize int, batchWait time.Duration, retryDelay time.Duration, workerID int) { reader := kafka.NewReader(realtimeReaderConfig(brokers, kafkaTopics)) defer reader.Close() workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)} registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 1) defer registry.SetGauge("vehicle_realtime_worker_active", workerLabels, 0) logger.Info("realtime kafka consumer started", "worker", workerID, "topics", strings.Join(kafkaTopics, ","), "batch_size", batchSize, "batch_wait_ms", batchWait.Milliseconds()) for { message, err := reader.FetchMessage(ctx) if err != nil { if ctx.Err() != nil { return } logger.Error("kafka fetch failed", "error", err) continue } batch := collectRealtimeBatch(ctx, reader, message, batchSize, batchWait) processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, reader, batch, retryDelay, workerLabels) } } func validateRealtimeConsumerConfig(kafkaTopics []string, workers int) error { if len(kafkaTopics) == 0 { return errors.New("KAFKA_TOPICS must include raw topics") } for _, topic := range kafkaTopics { if _, ok := topics.ProtocolForKnownRawTopic(topic); !ok { return fmt.Errorf("realtime-writer consumes known raw topics only, got %q", topic) } } if workers <= 0 { return errors.New("REALTIME_WORKERS must be greater than zero") } return nil } func realtimeReaderConfig(brokers []string, kafkaTopics []string) kafka.ReaderConfig { return kafka.ReaderConfig{ Brokers: brokers, GroupID: env("KAFKA_GROUP", "go-realtime-api"), GroupTopics: kafkaTopics, MinBytes: 1, MaxBytes: 10e6, StartOffset: kafka.LastOffset, } } func kafkaTopicsFromEnv() []string { defaultTopics := strings.Join([]string{topics.RawGB32960, topics.RawJT808, topics.RawYutongMQTT}, ",") return splitCSV(env("KAFKA_TOPICS", defaultTopics)) } const kafkaMessageOperationTimeout = 30 * time.Second type realtimeUpdater interface { Update(context.Context, envelope.FrameEnvelope) error } type realtimeBatchUpdater interface { UpdateBatch(context.Context, []envelope.FrameEnvelope) error } type fastRealtimeUpdater struct { repository *realtime.Repository } func (u fastRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { return u.repository.FastUpdate(ctx, env) } func (u fastRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { if u.repository == nil || len(envs) == 0 { return nil } return u.repository.FastUpdateBatch(ctx, envs) } type storeMetricUpdater struct { store string delegate realtimeUpdater registry *metrics.Registry } var realtimeStoreUpdateDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} func (u storeMetricUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { if u.delegate == nil { return nil } started := time.Now() err := u.delegate.Update(ctx, env) elapsed := time.Since(started) status := "ok" if err != nil { status = "error" } if u.registry != nil { labels := metrics.Labels{ "store": u.store, "protocol": string(env.Protocol), "status": status, } elapsedMS := float64(elapsed.Milliseconds()) u.registry.IncCounter("vehicle_realtime_store_updates_total", labels) u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS) u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS) metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels) } return err } func (u storeMetricUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { if u.delegate == nil || len(envs) == 0 { return nil } batchDelegate, ok := u.delegate.(realtimeBatchUpdater) if !ok { for _, env := range envs { if err := u.Update(ctx, env); err != nil { return err } } return nil } started := time.Now() err := batchDelegate.UpdateBatch(ctx, envs) elapsed := time.Since(started) status := statusFromError(err) if u.registry != nil { elapsedMS := float64(elapsed.Milliseconds()) counts := realtimeProtocolCounts(envs) for protocol, count := range counts { labels := metrics.Labels{ "store": u.store, "protocol": string(protocol), "status": status, } u.registry.AddCounter("vehicle_realtime_store_updates_total", labels, float64(count)) u.registry.IncCounter("vehicle_realtime_store_batch_updates_total", labels) u.registry.SetGauge("vehicle_realtime_store_update_duration_ms", labels, elapsedMS) u.registry.ObserveHistogram("vehicle_realtime_store_update_duration_ms_histogram", labels, realtimeStoreUpdateDurationBucketsMS, elapsedMS) metrics.RecordLastActivity(u.registry, "vehicle_realtime_last_store_update_unix_seconds", labels) } } return err } func realtimeProtocolCounts(envs []envelope.FrameEnvelope) map[envelope.Protocol]int { counts := make(map[envelope.Protocol]int) for _, env := range envs { counts[env.Protocol]++ } return counts } type retryRealtimeUpdater struct { delegate realtimeUpdater attempts int delay time.Duration } func (u retryRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { if u.delegate == nil { return nil } attempts := u.attempts if attempts <= 0 { attempts = 1 } var err error for attempt := 1; attempt <= attempts; attempt++ { err = u.delegate.Update(ctx, env) if err == nil || !isTransientMySQLWriteError(err) || attempt == attempts { return err } if u.delay <= 0 { continue } timer := time.NewTimer(u.delay) select { case <-ctx.Done(): timer.Stop() return ctx.Err() case <-timer.C: } } return err } func (u retryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { if u.delegate == nil || len(envs) == 0 { return nil } attempts := u.attempts if attempts <= 0 { attempts = 1 } var err error for attempt := 1; attempt <= attempts; attempt++ { err = updateRealtimeBatch(ctx, u.delegate, envs) if err == nil || !isTransientMySQLWriteError(err) || attempt == attempts { return err } if u.delay <= 0 { continue } timer := time.NewTimer(u.delay) select { case <-ctx.Done(): timer.Stop() return ctx.Err() case <-timer.C: } } return err } type throttledRealtimeUpdater struct { delegate realtimeUpdater interval time.Duration store string registry *metrics.Registry cacheRetention time.Duration cleanupInterval time.Duration mu sync.Mutex lastMS map[string]int64 lastCleanupMS int64 lastDeleted int } func newThrottledRealtimeUpdater(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry) realtimeUpdater { return newThrottledRealtimeUpdaterWithCache(delegate, interval, store, registry, defaultThrottleCacheRetention, defaultThrottleCacheCleanupInterval) } func newThrottledRealtimeUpdaterWithCache(delegate realtimeUpdater, interval time.Duration, store string, registry *metrics.Registry, cacheRetention time.Duration, cleanupInterval time.Duration) realtimeUpdater { if delegate == nil || interval <= 0 { return delegate } if cacheRetention < 0 { cacheRetention = 0 } if cleanupInterval < 0 { cleanupInterval = 0 } return &throttledRealtimeUpdater{ delegate: delegate, interval: interval, store: store, registry: registry, cacheRetention: cacheRetention, cleanupInterval: cleanupInterval, lastMS: map[string]int64{}, } } func (u *throttledRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { if u.delegate == nil { return nil } pass, status := u.shouldPass(env) u.record(status, env) u.recordCacheMetrics() if !pass { return nil } return u.delegate.Update(ctx, env) } func (u *throttledRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { for _, env := range envs { if err := u.Update(ctx, env); err != nil { return err } } return nil } func (u *throttledRealtimeUpdater) shouldPass(env envelope.FrameEnvelope) (bool, string) { if !envelope.IsRealtimeTelemetryFrame(env) { return false, "skipped_non_realtime" } vin := strings.TrimSpace(env.VIN) if vin == "" { return false, "skipped_missing_vin" } eventMS, ok := envelope.NormalizedEventTimeMS(env) if !ok || eventMS <= 0 { eventMS = env.ReceivedAtMS } if eventMS <= 0 { eventMS = time.Now().UnixMilli() } key := string(env.Protocol) + "\x00" + vin intervalMS := u.interval.Milliseconds() u.mu.Lock() defer u.mu.Unlock() if u.lastMS == nil { u.lastMS = map[string]int64{} } u.cleanupLocked(throttleCleanupTimeMS(env, eventMS)) if last := u.lastMS[key]; last > 0 && eventMS < last+intervalMS { return false, "skipped_interval" } u.lastMS[key] = eventMS return true, "passed" } func (u *throttledRealtimeUpdater) cleanupLocked(nowMS int64) { if u.cacheRetention <= 0 || nowMS <= 0 { return } intervalMS := u.cleanupInterval.Milliseconds() if intervalMS > 0 && u.lastCleanupMS > 0 && nowMS < u.lastCleanupMS+intervalMS { return } u.lastCleanupMS = nowMS cutoff := nowMS - u.cacheRetention.Milliseconds() deleted := 0 for key, last := range u.lastMS { if last > 0 && last < cutoff { delete(u.lastMS, key) deleted++ } } u.lastDeleted = deleted } func throttleCleanupTimeMS(env envelope.FrameEnvelope, fallbackMS int64) int64 { if env.ReceivedAtMS > 0 { return env.ReceivedAtMS } if fallbackMS > 0 { return fallbackMS } return time.Now().UnixMilli() } func (u *throttledRealtimeUpdater) record(status string, env envelope.FrameEnvelope) { if u.registry == nil { return } u.registry.IncCounter("vehicle_realtime_throttle_total", metrics.Labels{ "store": u.store, "protocol": string(env.Protocol), "status": status, }) } func (u *throttledRealtimeUpdater) recordCacheMetrics() { if u.registry == nil { return } u.mu.Lock() entries := len(u.lastMS) lastDeleted := u.lastDeleted lastCleanupMS := u.lastCleanupMS u.mu.Unlock() labels := metrics.Labels{"store": u.store} u.registry.SetGauge("vehicle_realtime_throttle_cache_entries", labels, float64(entries)) u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_deleted", labels, float64(lastDeleted)) if lastCleanupMS > 0 { u.registry.SetGauge("vehicle_realtime_throttle_cache_last_cleanup_unix_seconds", labels, float64(lastCleanupMS)/1000) } } func isTransientMySQLWriteError(err error) bool { if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return false } text := strings.ToLower(strings.TrimSpace(err.Error())) return strings.Contains(text, "deadlock") || strings.Contains(text, "error 1213") || strings.Contains(text, "40001") || strings.Contains(text, "lock wait timeout") || strings.Contains(text, "error 1205") || strings.Contains(text, "timeout") || strings.Contains(text, "temporary") || strings.Contains(text, "temporarily") || strings.Contains(text, "connection refused") || strings.Contains(text, "connection reset") || strings.Contains(text, "connection closed") || strings.Contains(text, "broken pipe") || strings.Contains(text, "bad connection") || strings.Contains(text, "invalid connection") || strings.Contains(text, "i/o timeout") || text == "eof" || strings.Contains(text, "unexpected eof") || strings.Contains(text, "server is down") || strings.Contains(text, "network is unreachable") || strings.Contains(text, "no route to host") } type compositeRealtimeUpdater struct { primary realtimeUpdater secondary realtimeUpdater } func composeRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater) realtimeUpdater { if primary == nil { return secondary } if secondary == nil { return primary } return compositeRealtimeUpdater{primary: primary, secondary: secondary} } func (u compositeRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { if u.primary != nil { if err := u.primary.Update(ctx, env); err != nil { return err } } if u.secondary == nil { return nil } return u.secondary.Update(ctx, env) } func (u compositeRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { if u.primary != nil { if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil { return err } } return updateRealtimeBatch(ctx, u.secondary, envs) } type asyncSecondaryRealtimeUpdater struct { primary realtimeUpdater secondary realtimeUpdater queue chan envelope.FrameEnvelope registry *metrics.Registry logger realtimeLogger closeOnce sync.Once closed chan struct{} wg sync.WaitGroup } func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater, queueSize int, workers int, registry *metrics.Registry, logger realtimeLogger) *asyncSecondaryRealtimeUpdater { if queueSize <= 0 { queueSize = 20000 } if workers <= 0 { workers = 4 } updater := &asyncSecondaryRealtimeUpdater{ primary: primary, secondary: secondary, queue: make(chan envelope.FrameEnvelope, queueSize), registry: registry, logger: logger, closed: make(chan struct{}), } updater.recordQueueCapacity() updater.wg.Add(workers) for i := 0; i < workers; i++ { go updater.run() } return updater } func (u *asyncSecondaryRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error { if u.primary != nil { if err := u.primary.Update(ctx, env); err != nil { return err } } if u.secondary == nil { return nil } select { case <-u.closed: u.recordQueueMetric(env, "closed") u.recordQueueDepth(env) return nil default: } select { case u.queue <- env: u.recordQueueMetric(env, "queued") u.recordQueueDepth(env) return nil case <-u.closed: u.recordQueueMetric(env, "closed") u.recordQueueDepth(env) return nil default: u.recordQueueMetric(env, "dropped") u.recordQueueDepth(env) return nil } } func (u *asyncSecondaryRealtimeUpdater) UpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error { if u.primary != nil { if err := updateRealtimeBatch(ctx, u.primary, envs); err != nil { return err } } if u.secondary == nil { return nil } for _, env := range envs { if err := u.enqueueSecondary(env); err != nil { return err } } return nil } func (u *asyncSecondaryRealtimeUpdater) enqueueSecondary(env envelope.FrameEnvelope) error { select { case <-u.closed: u.recordQueueMetric(env, "closed") u.recordQueueDepth(env) return nil default: } select { case u.queue <- env: u.recordQueueMetric(env, "queued") u.recordQueueDepth(env) return nil case <-u.closed: u.recordQueueMetric(env, "closed") u.recordQueueDepth(env) return nil default: u.recordQueueMetric(env, "dropped") u.recordQueueDepth(env) return nil } } func (u *asyncSecondaryRealtimeUpdater) run() { defer u.wg.Done() for { select { case env := <-u.queue: u.updateSecondary(env) case <-u.closed: for { select { case env := <-u.queue: u.updateSecondary(env) default: return } } } } } func (u *asyncSecondaryRealtimeUpdater) updateSecondary(env envelope.FrameEnvelope) { u.recordQueueDepth(env) secondaryCtx, cancel := context.WithTimeout(context.Background(), kafkaMessageOperationTimeout) if err := u.secondary.Update(secondaryCtx, env); err != nil && u.logger != nil { u.logger.Error("async realtime secondary update failed", "protocol", env.Protocol, "vin", env.VIN, "event_id", env.StableEventID(), "error", err) } cancel() u.recordQueueDepth(env) } func (u *asyncSecondaryRealtimeUpdater) recordQueueMetric(env envelope.FrameEnvelope, status string) { if u.registry == nil { return } u.registry.IncCounter("vehicle_realtime_async_queue_total", metrics.Labels{ "store": "mysql", "protocol": string(env.Protocol), "status": status, }) } func (u *asyncSecondaryRealtimeUpdater) recordQueueDepth(env envelope.FrameEnvelope) { if u.registry == nil { return } u.registry.SetGauge("vehicle_realtime_async_queue_depth", metrics.Labels{ "store": "mysql", "protocol": string(env.Protocol), }, float64(len(u.queue))) } func (u *asyncSecondaryRealtimeUpdater) recordQueueCapacity() { if u.registry == nil { return } u.registry.SetGauge("vehicle_realtime_async_queue_capacity", metrics.Labels{ "store": "mysql", }, float64(cap(u.queue))) } func (u *asyncSecondaryRealtimeUpdater) Close() { u.closeOnce.Do(func() { close(u.closed) }) u.wg.Wait() } type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } type kafkaMessageFetcher interface { FetchMessage(context.Context) (kafka.Message, error) } type realtimeBatchItem struct { message kafka.Message env envelope.FrameEnvelope valid bool processed bool } type realtimeBatchOutcome struct { commitMessages []kafka.Message retryMessages []kafka.Message } func collectRealtimeBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { if maxSize <= 1 { return []kafka.Message{first} } if maxWait <= 0 { maxWait = 20 * time.Millisecond } batch := []kafka.Message{first} deadline := time.Now().Add(maxWait) for len(batch) < maxSize { remaining := time.Until(deadline) if remaining <= 0 { break } fetchCtx, cancel := context.WithTimeout(ctx, remaining) message, err := fetcher.FetchMessage(fetchCtx) cancel() if err != nil { break } batch = append(batch, message) } return batch } func processRealtimeBatch(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message) realtimeBatchOutcome { return processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, messages, nil) } func processRealtimeBatchForWorker(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) realtimeBatchOutcome { if len(messages) == 0 { return realtimeBatchOutcome{} } messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) defer cancel() setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), 0) defer setRealtimeBatchPendingForWorker(registry, workerLabels, 0, 0) items := make([]realtimeBatchItem, 0, len(messages)) envs := make([]envelope.FrameEnvelope, 0, len(messages)) for _, message := range messages { itemIndex := len(items) items = append(items, realtimeBatchItem{message: message}) addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "received") addRealtimeLagMetric(registry, message) var env envelope.FrameEnvelope if err := json.Unmarshal(message.Value, &env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json") logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) items[itemIndex].processed = true continue } if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status) logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) items[itemIndex].processed = true continue } if status := realtimeProjectionSkipStatus(env); status != "" { addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status) items[itemIndex].processed = true continue } items[itemIndex].env = env items[itemIndex].valid = true envs = append(envs, env) } setRealtimeBatchPendingForWorker(registry, workerLabels, len(messages), len(envs)) if len(envs) > 0 { started := time.Now() err := updateRealtimeBatch(messageCtx, updater, envs) recordRealtimeBatchFlush(registry, statusFromError(err), len(messages), len(envs), time.Since(started)) if err != nil { logger.Error("realtime batch update failed; falling back to per-message isolation", "messages", len(messages), "valid_messages", len(envs), "error", err) if !fallbackRealtimeBatch(messageCtx, logger, registry, updater, items) { committed, commitErr := commitRealtimeProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items) return realtimeFailureOutcome(messages, committed, commitErr) } } else { for index := range items { if items[index].valid { items[index].processed = true addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok") } } } } if err := committer.CommitMessages(messageCtx, messages...); err != nil { for _, message := range messages { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") } first := messages[0] logger.Error("kafka batch commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) return realtimeBatchOutcome{commitMessages: messages} } for _, message := range messages { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") } return realtimeBatchOutcome{} } func realtimeFailureOutcome(messages []kafka.Message, committed []kafka.Message, commitErr error) realtimeBatchOutcome { outcome := realtimeBatchOutcome{ retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed), } if commitErr != nil { outcome.commitMessages = committed } return outcome } func processRealtimeBatchReliably(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) { processRealtimeBatchReliablyForWorker(ctx, logger, registry, updater, committer, messages, retryDelay, nil) } func processRealtimeBatchReliablyForWorker(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) { defer registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, 0) pending := messages for len(pending) > 0 { outcome := processRealtimeBatchForWorker(ctx, logger, registry, updater, committer, pending, workerLabels) if len(outcome.commitMessages) > 0 { registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"}) if !retryRealtimeCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) { return } } pending = outcome.retryMessages registry.SetGauge("vehicle_realtime_retry_pending_messages", workerLabels, float64(len(pending))) if len(pending) == 0 { return } registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "write_error"}) if !waitForRealtimeRetry(ctx, retryDelay) { return } } } func retryRealtimeCommit(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool { for len(messages) > 0 { operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) err := committer.CommitMessages(operationCtx, messages...) cancel() if err == nil { for _, message := range messages { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") } return true } for _, message := range messages { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") } first := messages[0] logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err) registry.IncCounter("vehicle_realtime_batch_retries_total", metrics.Labels{"reason": "commit_error"}) if !waitForRealtimeRetry(ctx, retryDelay) { return false } } return true } func waitForRealtimeRetry(ctx context.Context, retryDelay time.Duration) bool { if retryDelay <= 0 { retryDelay = 100 * time.Millisecond } timer := time.NewTimer(retryDelay) defer timer.Stop() select { case <-ctx.Done(): return false case <-timer.C: return true } } func fallbackRealtimeBatch(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, items []realtimeBatchItem) bool { for index := range items { if !items[index].valid { continue } if err := updater.Update(ctx, items[index].env); err != nil { message := items[index].message addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error") logger.Error("realtime fallback update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", items[index].env.StableEventID(), "error", err) return false } items[index].processed = true addRealtimeMetric(registry, "vehicle_realtime_updates_total", items[index].message, "ok") } return true } func commitRealtimeProcessedPrefixAfterFailure(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, committer kafkaMessageCommitter, items []realtimeBatchItem) ([]kafka.Message, error) { committable := realtimeProcessedPrefixMessagesByPartition(items) if len(committable) == 0 { return nil, nil } if err := committer.CommitMessages(ctx, committable...); err != nil { for _, message := range committable { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") } first := committable[0] logger.Error("kafka realtime processed-prefix commit failed after batch update failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err) return committable, err } for _, message := range committable { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") } return committable, nil } func realtimeProcessedPrefixMessagesByPartition(items []realtimeBatchItem) []kafka.Message { type partitionKey struct { topic string partition int } groups := map[partitionKey][]realtimeBatchItem{} for _, item := range items { key := partitionKey{topic: item.message.Topic, partition: item.message.Partition} groups[key] = append(groups[key], item) } var keys []partitionKey for key := range groups { keys = append(keys, key) } sort.Slice(keys, func(i, j int) bool { if keys[i].topic != keys[j].topic { return keys[i].topic < keys[j].topic } return keys[i].partition < keys[j].partition }) var out []kafka.Message for _, key := range keys { group := groups[key] sort.Slice(group, func(i, j int) bool { return group[i].message.Offset < group[j].message.Offset }) var previousOffset int64 for index, item := range group { if index > 0 && item.message.Offset != previousOffset+1 { break } if !item.processed { break } out = append(out, item.message) previousOffset = item.message.Offset } } return out } func updateRealtimeBatch(ctx context.Context, updater realtimeUpdater, envs []envelope.FrameEnvelope) error { if updater == nil || len(envs) == 0 { return nil } if batchUpdater, ok := updater.(realtimeBatchUpdater); ok { return batchUpdater.UpdateBatch(ctx, envs) } for _, env := range envs { if err := updater.Update(ctx, env); err != nil { return err } } return nil } func processRealtimeMessage(ctx context.Context, logger interface { Error(string, ...any) Warn(string, ...any) }, registry *metrics.Registry, updater realtimeUpdater, committer kafkaMessageCommitter, message kafka.Message) { messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout) defer cancel() addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "received") addRealtimeLagMetric(registry, message) var env envelope.FrameEnvelope if err := json.Unmarshal(message.Value, &env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, "invalid_json") logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) if err := committer.CommitMessages(messageCtx, message); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") logger.Error("kafka commit invalid envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") return } if status, err := topics.ValidateRawEnvelope(message.Topic, env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_messages_total", message, status) logger.Warn("skip mismatched realtime raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err) if err := committer.CommitMessages(messageCtx, message); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") logger.Error("kafka commit mismatched realtime raw envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") return } if status := realtimeProjectionSkipStatus(env); status != "" { addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, status) if err := committer.CommitMessages(messageCtx, message); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") logger.Error("kafka commit skipped realtime envelope failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "status", status, "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") return } if err := updater.Update(messageCtx, env); err != nil { addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "error") logger.Error("realtime projector update failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_updates_total", message, "ok") if err := committer.CommitMessages(messageCtx, message); err != nil { addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "error") logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err) return } addRealtimeMetric(registry, "vehicle_realtime_kafka_commits_total", message, "ok") } func realtimeProjectionSkipStatus(env envelope.FrameEnvelope) string { if !envelope.IsRealtimeTelemetryFrame(env) { return "skipped_non_realtime" } if strings.TrimSpace(env.VIN) == "" { return "skipped_missing_vin" } vehicleKey := strings.TrimSpace(env.VehicleKey()) if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") { return "skipped_missing_vehicle_key" } if len(env.ParsedFields) == 0 { return "skipped_missing_fields" } return "" } func addRealtimeMetric(registry *metrics.Registry, name string, message kafka.Message, status string) { if registry == nil { return } labels := metrics.Labels{ "topic": message.Topic, "status": status, } registry.IncCounter(name, labels) switch name { case "vehicle_realtime_kafka_messages_total": metrics.RecordLastActivity(registry, "vehicle_realtime_last_message_unix_seconds", labels) case "vehicle_realtime_updates_total": metrics.RecordLastActivity(registry, "vehicle_realtime_last_update_unix_seconds", labels) case "vehicle_realtime_kafka_commits_total": metrics.RecordLastActivity(registry, "vehicle_realtime_last_commit_unix_seconds", labels) } } func recordPlateCacheStats(registry *metrics.Registry, stats realtime.PlateCacheStats) { if registry == nil { return } registry.SetGauge("vehicle_realtime_plate_cache_entries", nil, float64(stats.Entries)) registry.SetGauge("vehicle_realtime_plate_cache_max_entries", nil, float64(stats.MaxEntries)) registry.SetGauge("vehicle_realtime_plate_cache_evictions_total", nil, float64(stats.Evictions)) } func addRealtimeLagMetric(registry *metrics.Registry, message kafka.Message) { if registry == nil { return } registry.SetKafkaLag("vehicle_realtime_kafka_lag", message.Topic, message.Partition, message.Offset, message.HighWaterMark) } var realtimeBatchFlushDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} func recordRealtimeBatchFlush(registry *metrics.Registry, status string, messages int, validMessages int, elapsed time.Duration) { if registry == nil { return } labels := metrics.Labels{"status": status} elapsedMS := float64(elapsed.Milliseconds()) registry.IncCounter("vehicle_realtime_batch_flush_total", labels) registry.AddCounter("vehicle_realtime_batch_messages_total", labels, float64(messages)) registry.AddCounter("vehicle_realtime_batch_valid_messages_total", labels, float64(validMessages)) registry.SetGauge("vehicle_realtime_batch_flush_duration_ms", labels, elapsedMS) registry.ObserveHistogram("vehicle_realtime_batch_flush_duration_ms_histogram", labels, realtimeBatchFlushDurationBucketsMS, elapsedMS) } func setRealtimeBatchPending(registry *metrics.Registry, messages int, validMessages int) { setRealtimeBatchPendingForWorker(registry, nil, messages, validMessages) } func setRealtimeBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int, validMessages int) { if registry == nil { return } registry.SetGauge("vehicle_realtime_batch_pending_messages", workerLabels, float64(messages)) registry.SetGauge("vehicle_realtime_batch_pending_valid_messages", workerLabels, float64(validMessages)) } func statusFromError(err error) string { if err != nil { return "error" } return "ok" } func realtimeRoleFromEnv() string { return normalizeRealtimeRole(env("REALTIME_ROLE", "combined")) } func normalizeRealtimeRole(value string) string { switch strings.ToLower(strings.TrimSpace(value)) { case "api", "query", "read", "readonly", "read-only": return "api" case "writer", "projector", "consumer", "write": return "writer" case "combined", "all", "both", "": return "combined" default: return "combined" } } func realtimeRoleAllowsAPI(role string) bool { switch normalizeRealtimeRole(role) { case "writer": return false default: return true } } func realtimeRoleAllowsWriter(role string) bool { switch normalizeRealtimeRole(role) { case "api": return false default: return true } } func realtimeRoleNeedsRedis(apiEnabled bool, writerEnabled bool, redisProjectorEnabled bool) bool { return apiEnabled || (writerEnabled && redisProjectorEnabled) } func realtimeServiceName(role string) string { if normalizeRealtimeRole(role) == "writer" { return "vehicle-realtime-writer" } return "vehicle-realtime-api" } func env(key string, fallback string) string { value := strings.TrimSpace(os.Getenv(key)) if value == "" { return fallback } return value } func envInt(key string, fallback int) int { value := strings.TrimSpace(os.Getenv(key)) if value == "" { return fallback } parsed, err := strconv.Atoi(value) if err != nil { return fallback } return parsed } func boolGauge(value bool) float64 { if value { return 1 } return 0 } func splitCSV(value string) []string { var out []string for _, item := range strings.Split(value, ",") { item = strings.TrimSpace(item) if item != "" { out = append(out, item) } } return out }