package main import ( "context" "fmt" "log" "os/signal" "sort" "strings" "syscall" "time" _ "github.com/go-sql-driver/mysql" "github.com/segmentio/kafka-go" "lingniu/vehicle-data-platform/apps/api/internal/config" "lingniu/vehicle-data-platform/apps/api/internal/platform" ) const alertStreamOperationTimeout = 30 * time.Second type alertStreamStore interface { RecordAlertStreamBatch(context.Context, string, []platform.AlertStreamRecord) (platform.AlertStreamBatchResult, error) } type kafkaMessageFetcher interface { FetchMessage(context.Context) (kafka.Message, error) } type kafkaMessageCommitter interface { CommitMessages(context.Context, ...kafka.Message) error } func main() { cfg := config.Load() if err := validateAlertStreamConfig(cfg); err != nil { log.Fatal(err) } ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN) if err != nil { log.Fatal(err) } defer db.Close() store := platform.NewProductionStore(db, nil, "").WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup) reader := kafka.NewReader(alertStreamReaderConfig(cfg)) defer reader.Close() log.Printf("alert stream evaluator started mode=%s group=%s topics=%s batch_size=%d batch_wait=%s lateness=%s", cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup, strings.Join(cfg.AlertStreamKafkaTopics, ","), cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait, cfg.AlertStreamLateness) for { message, err := reader.FetchMessage(ctx) if err != nil { if ctx.Err() != nil { log.Printf("alert stream evaluator stopped") return } log.Printf("alert stream fetch failed error=%v", err) continue } messages := collectAlertStreamBatch(ctx, reader, message, cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait) started := time.Now() operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertStreamOperationTimeout) records := make([]platform.AlertStreamRecord, 0, len(messages)) for _, item := range messages { records = append(records, platform.DecodeAlertStreamRecord(item.Topic, item.Partition, item.Offset, item.HighWaterMark, item.Value, cfg.AlertStreamLateness)) } invalidCodes := alertStreamInvalidCodes(records) result, recordErr := store.RecordAlertStreamBatch(operationCtx, cfg.AlertStreamKafkaGroup, records) if recordErr == nil { recordErr = reader.CommitMessages(operationCtx, messages...) } cancel() if recordErr != nil { log.Printf("alert stream batch failed fetched=%d duration=%s error=%v", len(messages), time.Since(started), recordErr) time.Sleep(time.Second) continue } log.Printf("alert stream batch completed mode=%s fetched=%d processed=%d valid=%d invalid=%d invalid_codes=%s late=%d replay_skipped=%d partitions=%d rules=%d candidates_advanced=%d duplicate_observations=%d late_observations=%d opened=%d recovered=%d duration=%s", cfg.AlertStreamMode, result.Fetched, result.Processed, result.Valid, result.Invalid, invalidCodes, result.Late, result.ReplaySkipped, result.Partitions, result.RulesEvaluated, result.CandidatesAdvanced, result.DuplicateObservations, result.LateObservations, result.Opened, result.Recovered, time.Since(started)) } } func alertStreamInvalidCodes(records []platform.AlertStreamRecord) string { counts := map[string]int{} for _, record := range records { if !record.Valid { counts[record.ErrorCode]++ } } if len(counts) == 0 { return "none" } keys := make([]string, 0, len(counts)) for key := range counts { keys = append(keys, key) } sort.Strings(keys) parts := make([]string, 0, len(keys)) for _, key := range keys { parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key])) } return strings.Join(parts, ",") } func alertStreamReaderConfig(cfg config.Config) kafka.ReaderConfig { return kafka.ReaderConfig{ Brokers: cfg.AlertStreamKafkaBrokers, GroupID: cfg.AlertStreamKafkaGroup, GroupTopics: cfg.AlertStreamKafkaTopics, MinBytes: 1, MaxBytes: 10e6, StartOffset: kafka.LastOffset, } } func validateAlertStreamConfig(cfg config.Config) error { if cfg.AlertStreamMode != "shadow" && cfg.AlertStreamMode != "active" { return fmt.Errorf("ALERT_STREAM_MODE must be shadow or active") } if cfg.MySQLDSN == "" { return fmt.Errorf("MYSQL_DSN is required") } if len(cfg.AlertStreamKafkaBrokers) == 0 || len(cfg.AlertStreamKafkaTopics) == 0 { return fmt.Errorf("ALERT_STREAM_KAFKA_BROKERS and ALERT_STREAM_KAFKA_TOPICS are required") } if strings.TrimSpace(cfg.AlertStreamKafkaGroup) == "" { return fmt.Errorf("ALERT_STREAM_KAFKA_GROUP is required") } if cfg.AlertStreamBatchSize < 1 || cfg.AlertStreamBatchSize > 1000 { return fmt.Errorf("ALERT_STREAM_BATCH_SIZE must be between 1 and 1000") } if cfg.AlertStreamBatchWait < time.Millisecond || cfg.AlertStreamBatchWait > 5*time.Second { return fmt.Errorf("ALERT_STREAM_BATCH_WAIT_MS must be between 1 and 5000") } if cfg.AlertStreamLateness < 0 || cfg.AlertStreamLateness > 24*time.Hour { return fmt.Errorf("ALERT_STREAM_LATENESS_SEC must be between 0 and 86400") } for _, topic := range cfg.AlertStreamKafkaTopics { if _, known := alertStreamTopicProtocol(topic); !known { return fmt.Errorf("unsupported alert stream topic %q", topic) } } return nil } func alertStreamTopicProtocol(topic string) (string, bool) { switch strings.TrimSpace(topic) { case "vehicle.fields.go.gb32960.v1": return "GB32960", true case "vehicle.fields.go.jt808.v1": return "JT808", true case "vehicle.fields.go.yutong-mqtt.v1": return "YUTONG_MQTT", true default: return "", false } } func collectAlertStreamBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message { if maxSize <= 1 { return []kafka.Message{first} } messages := []kafka.Message{first} deadline := time.Now().Add(maxWait) for len(messages) < maxSize { remaining := time.Until(deadline) if remaining <= 0 { break } fetchCtx, cancel := context.WithTimeout(ctx, remaining) message, err := fetcher.FetchMessage(fetchCtx) cancel() if err != nil { break } messages = append(messages, message) } return messages }