lingniu-vehicle-ingest/vehicle-data-platform/apps/api/cmd/alert-stream-evaluator/main.go

package main

import (
	"context"
	"fmt"
	"log"
	"os/signal"
	"sort"
	"strings"
	"syscall"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/segmentio/kafka-go"

	"lingniu/vehicle-data-platform/apps/api/internal/config"
	"lingniu/vehicle-data-platform/apps/api/internal/platform"
)

const alertStreamOperationTimeout = 30 * time.Second

type alertStreamStore interface {
	RecordAlertStreamBatch(context.Context, string, []platform.AlertStreamRecord) (platform.AlertStreamBatchResult, error)
}

type kafkaMessageFetcher interface {
	FetchMessage(context.Context) (kafka.Message, error)
}

type kafkaMessageCommitter interface {
	CommitMessages(context.Context, ...kafka.Message) error
}

func main() {
	cfg := config.Load()
	if err := validateAlertStreamConfig(cfg); err != nil {
		log.Fatal(err)
	}
	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer stop()
	db, err := platform.OpenSQL(ctx, "mysql", cfg.MySQLDSN)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	store := platform.NewProductionStore(db, nil, "").WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup)
	reader := kafka.NewReader(alertStreamReaderConfig(cfg))
	defer reader.Close()
	log.Printf("alert stream evaluator started mode=%s group=%s topics=%s batch_size=%d batch_wait=%s lateness=%s", cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup, strings.Join(cfg.AlertStreamKafkaTopics, ","), cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait, cfg.AlertStreamLateness)
	for {
		message, err := reader.FetchMessage(ctx)
		if err != nil {
			if ctx.Err() != nil {
				log.Printf("alert stream evaluator stopped")
				return
			}
			log.Printf("alert stream fetch failed error=%v", err)
			continue
		}
		messages := collectAlertStreamBatch(ctx, reader, message, cfg.AlertStreamBatchSize, cfg.AlertStreamBatchWait)
		started := time.Now()
		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), alertStreamOperationTimeout)
		records := make([]platform.AlertStreamRecord, 0, len(messages))
		for _, item := range messages {
			records = append(records, platform.DecodeAlertStreamRecord(item.Topic, item.Partition, item.Offset, item.HighWaterMark, item.Value, cfg.AlertStreamLateness))
		}
		invalidCodes := alertStreamInvalidCodes(records)
		result, recordErr := store.RecordAlertStreamBatch(operationCtx, cfg.AlertStreamKafkaGroup, records)
		if recordErr == nil {
			recordErr = reader.CommitMessages(operationCtx, messages...)
		}
		cancel()
		if recordErr != nil {
			log.Printf("alert stream batch failed fetched=%d duration=%s error=%v", len(messages), time.Since(started), recordErr)
			time.Sleep(time.Second)
			continue
		}
		log.Printf("alert stream batch completed mode=%s fetched=%d processed=%d valid=%d invalid=%d invalid_codes=%s late=%d replay_skipped=%d partitions=%d rules=%d candidates_advanced=%d duplicate_observations=%d late_observations=%d opened=%d recovered=%d duration=%s", cfg.AlertStreamMode, result.Fetched, result.Processed, result.Valid, result.Invalid, invalidCodes, result.Late, result.ReplaySkipped, result.Partitions, result.RulesEvaluated, result.CandidatesAdvanced, result.DuplicateObservations, result.LateObservations, result.Opened, result.Recovered, time.Since(started))
	}
}

func alertStreamInvalidCodes(records []platform.AlertStreamRecord) string {
	counts := map[string]int{}
	for _, record := range records {
		if !record.Valid {
			counts[record.ErrorCode]++
		}
	}
	if len(counts) == 0 {
		return "none"
	}
	keys := make([]string, 0, len(counts))
	for key := range counts {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	parts := make([]string, 0, len(keys))
	for _, key := range keys {
		parts = append(parts, fmt.Sprintf("%s:%d", key, counts[key]))
	}
	return strings.Join(parts, ",")
}

func alertStreamReaderConfig(cfg config.Config) kafka.ReaderConfig {
	return kafka.ReaderConfig{
		Brokers:     cfg.AlertStreamKafkaBrokers,
		GroupID:     cfg.AlertStreamKafkaGroup,
		GroupTopics: cfg.AlertStreamKafkaTopics,
		MinBytes:    1,
		MaxBytes:    10e6,
		StartOffset: kafka.LastOffset,
	}
}

func validateAlertStreamConfig(cfg config.Config) error {
	if cfg.AlertStreamMode != "shadow" && cfg.AlertStreamMode != "active" {
		return fmt.Errorf("ALERT_STREAM_MODE must be shadow or active")
	}
	if cfg.MySQLDSN == "" {
		return fmt.Errorf("MYSQL_DSN is required")
	}
	if len(cfg.AlertStreamKafkaBrokers) == 0 || len(cfg.AlertStreamKafkaTopics) == 0 {
		return fmt.Errorf("ALERT_STREAM_KAFKA_BROKERS and ALERT_STREAM_KAFKA_TOPICS are required")
	}
	if strings.TrimSpace(cfg.AlertStreamKafkaGroup) == "" {
		return fmt.Errorf("ALERT_STREAM_KAFKA_GROUP is required")
	}
	if cfg.AlertStreamBatchSize < 1 || cfg.AlertStreamBatchSize > 1000 {
		return fmt.Errorf("ALERT_STREAM_BATCH_SIZE must be between 1 and 1000")
	}
	if cfg.AlertStreamBatchWait < time.Millisecond || cfg.AlertStreamBatchWait > 5*time.Second {
		return fmt.Errorf("ALERT_STREAM_BATCH_WAIT_MS must be between 1 and 5000")
	}
	if cfg.AlertStreamLateness < 0 || cfg.AlertStreamLateness > 24*time.Hour {
		return fmt.Errorf("ALERT_STREAM_LATENESS_SEC must be between 0 and 86400")
	}
	for _, topic := range cfg.AlertStreamKafkaTopics {
		if _, known := alertStreamTopicProtocol(topic); !known {
			return fmt.Errorf("unsupported alert stream topic %q", topic)
		}
	}
	return nil
}

func alertStreamTopicProtocol(topic string) (string, bool) {
	switch strings.TrimSpace(topic) {
	case "vehicle.fields.go.gb32960.v1":
		return "GB32960", true
	case "vehicle.fields.go.jt808.v1":
		return "JT808", true
	case "vehicle.fields.go.yutong-mqtt.v1":
		return "YUTONG_MQTT", true
	default:
		return "", false
	}
}

func collectAlertStreamBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
	if maxSize <= 1 {
		return []kafka.Message{first}
	}
	messages := []kafka.Message{first}
	deadline := time.Now().Add(maxWait)
	for len(messages) < maxSize {
		remaining := time.Until(deadline)
		if remaining <= 0 {
			break
		}
		fetchCtx, cancel := context.WithTimeout(ctx, remaining)
		message, err := fetcher.FetchMessage(fetchCtx)
		cancel()
		if err != nil {
			break
		}
		messages = append(messages, message)
	}
	return messages
}