lingniu-vehicle-ingest/go/vehicle-gateway/cmd/fields-projector/main.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"log/slog"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/segmentio/kafka-go"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
)

func main() {
	logger := observability.NewLogger("vehicle-fields-projector")
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
	defer stop()

	cfg := loadConfig()
	if err := cfg.Validate(); err != nil {
		logger.Error("invalid fields projector config", "error", err)
		os.Exit(1)
	}
	if err := pingKafka(ctx, cfg.KafkaBrokers); err != nil {
		logger.Error("kafka connectivity check failed", "error", err)
		os.Exit(1)
	}
	registry := metrics.NewRegistry()
	recordConfigMetrics(registry, cfg)
	for _, route := range cfg.Routes {
		metrics.RegisterKafkaConsumerInfo(registry, "vehicle-fields-projector", route.GroupID, []string{route.RawTopic})
	}
	health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-fields-projector", nil, registry))

	logger.Info("fields projector started",
		"kafka_brokers", strings.Join(cfg.KafkaBrokers, ","),
		"group_prefix", cfg.GroupPrefix,
		"workers_per_protocol", cfg.WorkersPerProtocol,
		"batch_size", cfg.BatchSize,
		"batch_wait_ms", cfg.BatchWait.Milliseconds(),
		"operation_timeout_ms", cfg.OperationTimeout.Milliseconds(),
		"retry_delay_ms", cfg.RetryDelay.Milliseconds(),
		"start_offset", cfg.StartOffsetName)

	var workers sync.WaitGroup
	for _, route := range cfg.Routes {
		for workerID := 1; workerID <= cfg.WorkersPerProtocol; workerID++ {
			workers.Add(1)
			go func(route projectionRoute, workerID int) {
				defer workers.Done()
				runProjector(ctx, logger.With("protocol", route.Protocol, "worker", workerID), registry, cfg, route, workerID)
			}(route, workerID)
		}
	}
	workers.Wait()
}

type config struct {
	KafkaBrokers       []string
	GroupPrefix        string
	Routes             []projectionRoute
	WorkersPerProtocol int
	BatchSize          int
	BatchWait          time.Duration
	OperationTimeout   time.Duration
	RetryDelay         time.Duration
	StartOffset        int64
	StartOffsetName    string
}

type projectionRoute struct {
	Protocol    envelope.Protocol
	RawTopic    string
	FieldsTopic string
	GroupID     string
}

func loadConfig() config {
	groupPrefix := env("FIELDS_PROJECTOR_GROUP_PREFIX", "vehicle-fields-projector-v1")
	startOffsetName := strings.ToLower(env("KAFKA_START_OFFSET", "last"))
	startOffset := int64(kafka.LastOffset)
	if startOffsetName == "first" {
		startOffset = kafka.FirstOffset
	} else {
		startOffsetName = "last"
	}
	routes := []projectionRoute{
		{Protocol: envelope.ProtocolGB32960, RawTopic: env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960), FieldsTopic: env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)},
		{Protocol: envelope.ProtocolJT808, RawTopic: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808), FieldsTopic: env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)},
		{Protocol: envelope.ProtocolYutongMQTT, RawTopic: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT), FieldsTopic: env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)},
	}
	for index := range routes {
		routes[index].GroupID = groupPrefix + "-" + strings.ToLower(strings.ReplaceAll(string(routes[index].Protocol), "_", "-"))
	}
	return config{
		KafkaBrokers:       splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
		GroupPrefix:        groupPrefix,
		Routes:             routes,
		WorkersPerProtocol: envInt("FIELDS_PROJECTOR_WORKERS_PER_PROTOCOL", 1),
		BatchSize:          envInt("FIELDS_PROJECTOR_BATCH_SIZE", 500),
		BatchWait:          time.Duration(envInt("FIELDS_PROJECTOR_BATCH_WAIT_MS", 20)) * time.Millisecond,
		OperationTimeout:   time.Duration(envInt("FIELDS_PROJECTOR_OPERATION_TIMEOUT_MS", 30000)) * time.Millisecond,
		RetryDelay:         time.Duration(envInt("FIELDS_PROJECTOR_RETRY_DELAY_MS", 500)) * time.Millisecond,
		StartOffset:        startOffset,
		StartOffsetName:    startOffsetName,
	}
}

func (c config) Validate() error {
	if len(c.KafkaBrokers) == 0 {
		return errors.New("kafka brokers are required")
	}
	if strings.TrimSpace(c.GroupPrefix) == "" {
		return errors.New("fields projector group prefix is required")
	}
	if c.WorkersPerProtocol < 1 || c.BatchSize < 1 || c.BatchWait <= 0 || c.OperationTimeout <= 0 || c.RetryDelay <= 0 {
		return errors.New("fields projector worker, batch and timeout settings must be positive")
	}
	raw := make(map[string]string, len(c.Routes))
	fields := make(map[string]string, len(c.Routes))
	groups := map[string]struct{}{}
	for _, route := range c.Routes {
		protocol := string(route.Protocol)
		raw[protocol] = route.RawTopic
		fields[protocol] = route.FieldsTopic
		if strings.TrimSpace(route.GroupID) == "" {
			return fmt.Errorf("consumer group is required for protocol %s", route.Protocol)
		}
		if _, exists := groups[route.GroupID]; exists {
			return fmt.Errorf("duplicate consumer group %q", route.GroupID)
		}
		groups[route.GroupID] = struct{}{}
	}
	return topics.ValidateKafkaRawFields(raw, fields)
}

type kafkaBatchWriter interface {
	WriteMessages(context.Context, ...kafka.Message) error
}

type kafkaMessageFetcher interface {
	FetchMessage(context.Context) (kafka.Message, error)
}

type kafkaMessageCommitter interface {
	CommitMessages(context.Context, ...kafka.Message) error
}

func runProjector(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, cfg config, route projectionRoute, workerID int) {
	reader := kafka.NewReader(kafka.ReaderConfig{
		Brokers:     cfg.KafkaBrokers,
		GroupID:     route.GroupID,
		GroupTopics: []string{route.RawTopic},
		StartOffset: cfg.StartOffset,
		MinBytes:    1,
		MaxBytes:    10e6,
	})
	defer reader.Close()
	writer := &kafka.Writer{
		Addr:                   kafka.TCP(cfg.KafkaBrokers...),
		Balancer:               &kafka.Hash{},
		RequiredAcks:           kafka.RequireAll,
		AllowAutoTopicCreation: false,
		BatchTimeout:           cfg.BatchWait,
		Async:                  false,
	}
	defer writer.Close()

	labels := metrics.Labels{"protocol": string(route.Protocol), "worker": strconv.Itoa(workerID)}
	registry.SetGauge("vehicle_fields_projector_worker_active", labels, 1)
	defer registry.SetGauge("vehicle_fields_projector_worker_active", labels, 0)
	for {
		first, err := reader.FetchMessage(ctx)
		if err != nil {
			if ctx.Err() != nil {
				return
			}
			logger.Error("kafka fetch failed", "error", err)
			continue
		}
		batch := collectBatch(ctx, reader, first, cfg.BatchSize, cfg.BatchWait)
		processBatchReliably(ctx, logger, registry, writer, reader, route, batch, cfg.OperationTimeout, cfg.RetryDelay)
	}
}

func collectBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
	if maxSize <= 1 {
		return []kafka.Message{first}
	}
	batch := []kafka.Message{first}
	deadline := time.Now().Add(maxWait)
	for len(batch) < maxSize {
		remaining := time.Until(deadline)
		if remaining <= 0 {
			break
		}
		fetchCtx, cancel := context.WithTimeout(ctx, remaining)
		message, err := fetcher.FetchMessage(fetchCtx)
		cancel()
		if err != nil {
			break
		}
		batch = append(batch, message)
	}
	return batch
}

func processBatchReliably(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, writer kafkaBatchWriter, committer kafkaMessageCommitter, route projectionRoute, messages []kafka.Message, operationTimeout time.Duration, retryDelay time.Duration) {
	labels := metrics.Labels{"protocol": string(route.Protocol)}
	defer registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, 0)
	for len(messages) > 0 {
		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
		outputs, err := projectBatch(operationCtx, logger, registry, writer, route, messages)
		cancel()
		if err != nil {
			registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages)))
			registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "write_error"})
			if !waitForRetry(ctx, retryDelay) {
				return
			}
			continue
		}
		commitCtx, commitCancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
		err = committer.CommitMessages(commitCtx, messages...)
		commitCancel()
		if err == nil {
			for _, message := range messages {
				recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok")
			}
			_ = outputs
			return
		}
		for _, message := range messages {
			recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error")
		}
		registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages)))
		registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "commit_error"})
		logger.Error("kafka source offset commit failed", "topic", route.RawTopic, "messages", len(messages), "error", err)
		if !retryCommit(ctx, logger, registry, committer, messages, route.Protocol, operationTimeout, retryDelay) {
			return
		}
		return
	}
}

func projectBatch(ctx context.Context, logger interface {
	Error(string, ...any)
	Warn(string, ...any)
}, registry *metrics.Registry, writer kafkaBatchWriter, route projectionRoute, messages []kafka.Message) (int, error) {
	if len(messages) == 0 {
		return 0, nil
	}
	outputs := make([]kafka.Message, 0, len(messages))
	setBatchPending(registry, route.Protocol, len(messages), 0)
	defer setBatchPending(registry, route.Protocol, -len(messages), -len(outputs))
	for _, message := range messages {
		recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "received")
		recordKafkaLag(registry, message)
		var raw envelope.FrameEnvelope
		if err := json.Unmarshal(message.Value, &raw); err != nil {
			recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "invalid_json")
			logger.Warn("skip invalid raw envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
			continue
		}
		if status, err := topics.ValidateRawEnvelope(route.RawTopic, raw); err != nil {
			recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, status)
			logger.Warn("skip mismatched raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", raw.Protocol, "event_id", raw.StableEventID(), "error", err)
			continue
		}
		fields, ok := realtime.BuildFieldsEnvelope(raw)
		if !ok {
			status := "skipped_missing_fields"
			if !envelope.IsRealtimeTelemetryFrame(raw) {
				status = "skipped_non_realtime"
			}
			recordProjectionMetric(registry, route.Protocol, status, 0)
			continue
		}
		if status, err := topics.ValidateFieldsEnvelope(route.FieldsTopic, fields); err != nil {
			recordProjectionMetric(registry, route.Protocol, status, len(fields.Fields))
			logger.Warn("skip invalid projected fields envelope", "topic", route.FieldsTopic, "protocol", fields.Protocol, "event_id", fields.StableEventID(), "error", err)
			continue
		}
		payload, err := fields.MarshalJSONBytes()
		if err != nil {
			recordProjectionMetric(registry, route.Protocol, "marshal_error", len(fields.Fields))
			logger.Warn("skip fields envelope marshal error", "topic", route.FieldsTopic, "event_id", fields.StableEventID(), "error", err)
			continue
		}
		outputs = append(outputs, kafka.Message{
			Topic: route.FieldsTopic,
			Key:   fields.KafkaKey(),
			Value: payload,
			Time:  message.Time,
		})
		recordProjectionMetric(registry, route.Protocol, "projected", len(fields.Fields))
	}
	setBatchPending(registry, route.Protocol, 0, len(outputs))
	defer setBatchPending(registry, route.Protocol, 0, -len(outputs))
	if len(outputs) == 0 {
		return 0, nil
	}
	started := time.Now()
	err := writer.WriteMessages(ctx, outputs...)
	status := "ok"
	if err != nil {
		status = "error"
	}
	recordWriteDuration(registry, route.FieldsTopic, status, time.Since(started))
	for range outputs {
		recordMessageMetric(registry, "vehicle_fields_projector_kafka_writes_total", route.FieldsTopic, status)
	}
	if err != nil {
		logger.Error("fields kafka write failed", "topic", route.FieldsTopic, "messages", len(outputs), "error", err)
		return len(outputs), err
	}
	return len(outputs), nil
}

func retryCommit(ctx context.Context, logger interface {
	Error(string, ...any)
}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, protocol envelope.Protocol, operationTimeout, retryDelay time.Duration) bool {
	for {
		if !waitForRetry(ctx, retryDelay) {
			return false
		}
		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
		err := committer.CommitMessages(operationCtx, messages...)
		cancel()
		if err == nil {
			for _, message := range messages {
				recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok")
			}
			return true
		}
		for _, message := range messages {
			recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error")
		}
		registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(protocol), "reason": "commit_error"})
		logger.Error("kafka source offset commit retry failed", "messages", len(messages), "error", err)
	}
}

func waitForRetry(ctx context.Context, delay time.Duration) bool {
	timer := time.NewTimer(delay)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return false
	case <-timer.C:
		return true
	}
}

func pingKafka(ctx context.Context, brokers []string) error {
	if len(brokers) == 0 {
		return errors.New("kafka broker is required")
	}
	checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()
	conn, err := kafka.DialContext(checkCtx, "tcp", brokers[0])
	if err != nil {
		return err
	}
	return conn.Close()
}

func recordConfigMetrics(registry *metrics.Registry, cfg config) {
	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "workers_per_protocol"}, float64(cfg.WorkersPerProtocol))
	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize))
	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_wait_ms"}, float64(cfg.BatchWait.Milliseconds()))
	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationTimeout.Milliseconds()))
}

func recordMessageMetric(registry *metrics.Registry, name, topic, status string) {
	if registry == nil {
		return
	}
	labels := metrics.Labels{"topic": topic, "status": status}
	registry.IncCounter(name, labels)
	switch name {
	case "vehicle_fields_projector_kafka_messages_total":
		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_message_unix_seconds", labels)
	case "vehicle_fields_projector_kafka_writes_total":
		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_write_unix_seconds", labels)
	case "vehicle_fields_projector_kafka_commits_total":
		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_commit_unix_seconds", labels)
	}
}

func recordProjectionMetric(registry *metrics.Registry, protocol envelope.Protocol, status string, fieldCount int) {
	if registry == nil {
		return
	}
	labels := metrics.Labels{"protocol": string(protocol), "status": status}
	registry.IncCounter("vehicle_fields_projector_projections_total", labels)
	if fieldCount > 0 {
		registry.SetGauge("vehicle_fields_projector_field_count", labels, float64(fieldCount))
	}
}

func recordKafkaLag(registry *metrics.Registry, message kafka.Message) {
	if registry == nil {
		return
	}
	lag := message.HighWaterMark - message.Offset - 1
	if lag < 0 {
		lag = 0
	}
	registry.SetGauge("vehicle_fields_projector_kafka_lag", metrics.Labels{
		"topic": message.Topic, "partition": strconv.Itoa(message.Partition),
	}, float64(lag))
}

var projectorPendingMu sync.Mutex
var projectorPendingMessages = map[envelope.Protocol]int{}
var projectorPendingFields = map[envelope.Protocol]int{}

func setBatchPending(registry *metrics.Registry, protocol envelope.Protocol, messagesDelta, fieldsDelta int) {
	if registry == nil {
		return
	}
	projectorPendingMu.Lock()
	projectorPendingMessages[protocol] += messagesDelta
	projectorPendingFields[protocol] += fieldsDelta
	messages := projectorPendingMessages[protocol]
	fields := projectorPendingFields[protocol]
	projectorPendingMu.Unlock()
	labels := metrics.Labels{"protocol": string(protocol)}
	registry.SetGauge("vehicle_fields_projector_batch_pending_messages", labels, float64(messages))
	registry.SetGauge("vehicle_fields_projector_batch_pending_fields", labels, float64(fields))
}

var projectorWriteBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}

func recordWriteDuration(registry *metrics.Registry, topic, status string, elapsed time.Duration) {
	if registry == nil {
		return
	}
	registry.ObserveHistogram("vehicle_fields_projector_write_duration_ms_histogram", metrics.Labels{
		"topic": topic, "status": status,
	}, projectorWriteBucketsMS, float64(elapsed.Milliseconds()))
}

func env(key, fallback string) string {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	return value
}

func envInt(key string, fallback int) int {
	value := strings.TrimSpace(os.Getenv(key))
	if value == "" {
		return fallback
	}
	parsed, err := strconv.Atoi(value)
	if err != nil {
		return fallback
	}
	return parsed
}

func splitCSV(value string) []string {
	var out []string
	for _, item := range strings.Split(value, ",") {
		item = strings.TrimSpace(item)
		if item != "" {
			out = append(out, item)
		}
	}
	return out
}