feat: build vehicle data platform and production pipeline

2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
--- a/go/vehicle-gateway/cmd/fields-projector/main.go
+++ b/go/vehicle-gateway/cmd/fields-projector/main.go
@@ -0,0 +1,490 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log/slog"
+	"os"
+	"os/signal"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
+)
+
+func main() {
+	logger := observability.NewLogger("vehicle-fields-projector")
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+
+	cfg := loadConfig()
+	if err := cfg.Validate(); err != nil {
+		logger.Error("invalid fields projector config", "error", err)
+		os.Exit(1)
+	}
+	if err := pingKafka(ctx, cfg.KafkaBrokers); err != nil {
+		logger.Error("kafka connectivity check failed", "error", err)
+		os.Exit(1)
+	}
+	registry := metrics.NewRegistry()
+	recordConfigMetrics(registry, cfg)
+	for _, route := range cfg.Routes {
+		metrics.RegisterKafkaConsumerInfo(registry, "vehicle-fields-projector", route.GroupID, []string{route.RawTopic})
+	}
+	health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-fields-projector", nil, registry))
+
+	logger.Info("fields projector started",
+		"kafka_brokers", strings.Join(cfg.KafkaBrokers, ","),
+		"group_prefix", cfg.GroupPrefix,
+		"workers_per_protocol", cfg.WorkersPerProtocol,
+		"batch_size", cfg.BatchSize,
+		"batch_wait_ms", cfg.BatchWait.Milliseconds(),
+		"operation_timeout_ms", cfg.OperationTimeout.Milliseconds(),
+		"retry_delay_ms", cfg.RetryDelay.Milliseconds(),
+		"start_offset", cfg.StartOffsetName)
+
+	var workers sync.WaitGroup
+	for _, route := range cfg.Routes {
+		for workerID := 1; workerID <= cfg.WorkersPerProtocol; workerID++ {
+			workers.Add(1)
+			go func(route projectionRoute, workerID int) {
+				defer workers.Done()
+				runProjector(ctx, logger.With("protocol", route.Protocol, "worker", workerID), registry, cfg, route, workerID)
+			}(route, workerID)
+		}
+	}
+	workers.Wait()
+}
+
+type config struct {
+	KafkaBrokers       []string
+	GroupPrefix        string
+	Routes             []projectionRoute
+	WorkersPerProtocol int
+	BatchSize          int
+	BatchWait          time.Duration
+	OperationTimeout   time.Duration
+	RetryDelay         time.Duration
+	StartOffset        int64
+	StartOffsetName    string
+}
+
+type projectionRoute struct {
+	Protocol    envelope.Protocol
+	RawTopic    string
+	FieldsTopic string
+	GroupID     string
+}
+
+func loadConfig() config {
+	groupPrefix := env("FIELDS_PROJECTOR_GROUP_PREFIX", "vehicle-fields-projector-v1")
+	startOffsetName := strings.ToLower(env("KAFKA_START_OFFSET", "last"))
+	startOffset := int64(kafka.LastOffset)
+	if startOffsetName == "first" {
+		startOffset = kafka.FirstOffset
+	} else {
+		startOffsetName = "last"
+	}
+	routes := []projectionRoute{
+		{Protocol: envelope.ProtocolGB32960, RawTopic: env("KAFKA_TOPIC_GB32960_RAW", topics.RawGB32960), FieldsTopic: env("KAFKA_TOPIC_GB32960_FIELDS", topics.FieldsGB32960)},
+		{Protocol: envelope.ProtocolJT808, RawTopic: env("KAFKA_TOPIC_JT808_RAW", topics.RawJT808), FieldsTopic: env("KAFKA_TOPIC_JT808_FIELDS", topics.FieldsJT808)},
+		{Protocol: envelope.ProtocolYutongMQTT, RawTopic: env("KAFKA_TOPIC_YUTONG_MQTT_RAW", topics.RawYutongMQTT), FieldsTopic: env("KAFKA_TOPIC_YUTONG_MQTT_FIELDS", topics.FieldsYutongMQTT)},
+	}
+	for index := range routes {
+		routes[index].GroupID = groupPrefix + "-" + strings.ToLower(strings.ReplaceAll(string(routes[index].Protocol), "_", "-"))
+	}
+	return config{
+		KafkaBrokers:       splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
+		GroupPrefix:        groupPrefix,
+		Routes:             routes,
+		WorkersPerProtocol: envInt("FIELDS_PROJECTOR_WORKERS_PER_PROTOCOL", 1),
+		BatchSize:          envInt("FIELDS_PROJECTOR_BATCH_SIZE", 500),
+		BatchWait:          time.Duration(envInt("FIELDS_PROJECTOR_BATCH_WAIT_MS", 20)) * time.Millisecond,
+		OperationTimeout:   time.Duration(envInt("FIELDS_PROJECTOR_OPERATION_TIMEOUT_MS", 30000)) * time.Millisecond,
+		RetryDelay:         time.Duration(envInt("FIELDS_PROJECTOR_RETRY_DELAY_MS", 500)) * time.Millisecond,
+		StartOffset:        startOffset,
+		StartOffsetName:    startOffsetName,
+	}
+}
+
+func (c config) Validate() error {
+	if len(c.KafkaBrokers) == 0 {
+		return errors.New("kafka brokers are required")
+	}
+	if strings.TrimSpace(c.GroupPrefix) == "" {
+		return errors.New("fields projector group prefix is required")
+	}
+	if c.WorkersPerProtocol < 1 || c.BatchSize < 1 || c.BatchWait <= 0 || c.OperationTimeout <= 0 || c.RetryDelay <= 0 {
+		return errors.New("fields projector worker, batch and timeout settings must be positive")
+	}
+	raw := make(map[string]string, len(c.Routes))
+	fields := make(map[string]string, len(c.Routes))
+	groups := map[string]struct{}{}
+	for _, route := range c.Routes {
+		protocol := string(route.Protocol)
+		raw[protocol] = route.RawTopic
+		fields[protocol] = route.FieldsTopic
+		if strings.TrimSpace(route.GroupID) == "" {
+			return fmt.Errorf("consumer group is required for protocol %s", route.Protocol)
+		}
+		if _, exists := groups[route.GroupID]; exists {
+			return fmt.Errorf("duplicate consumer group %q", route.GroupID)
+		}
+		groups[route.GroupID] = struct{}{}
+	}
+	return topics.ValidateKafkaRawFields(raw, fields)
+}
+
+type kafkaBatchWriter interface {
+	WriteMessages(context.Context, ...kafka.Message) error
+}
+
+type kafkaMessageFetcher interface {
+	FetchMessage(context.Context) (kafka.Message, error)
+}
+
+type kafkaMessageCommitter interface {
+	CommitMessages(context.Context, ...kafka.Message) error
+}
+
+func runProjector(ctx context.Context, logger *slog.Logger, registry *metrics.Registry, cfg config, route projectionRoute, workerID int) {
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers:     cfg.KafkaBrokers,
+		GroupID:     route.GroupID,
+		GroupTopics: []string{route.RawTopic},
+		StartOffset: cfg.StartOffset,
+		MinBytes:    1,
+		MaxBytes:    10e6,
+	})
+	defer reader.Close()
+	writer := &kafka.Writer{
+		Addr:                   kafka.TCP(cfg.KafkaBrokers...),
+		Balancer:               &kafka.Hash{},
+		RequiredAcks:           kafka.RequireAll,
+		AllowAutoTopicCreation: false,
+		BatchTimeout:           cfg.BatchWait,
+		Async:                  false,
+	}
+	defer writer.Close()
+
+	labels := metrics.Labels{"protocol": string(route.Protocol), "worker": strconv.Itoa(workerID)}
+	registry.SetGauge("vehicle_fields_projector_worker_active", labels, 1)
+	defer registry.SetGauge("vehicle_fields_projector_worker_active", labels, 0)
+	for {
+		first, err := reader.FetchMessage(ctx)
+		if err != nil {
+			if ctx.Err() != nil {
+				return
+			}
+			logger.Error("kafka fetch failed", "error", err)
+			continue
+		}
+		batch := collectBatch(ctx, reader, first, cfg.BatchSize, cfg.BatchWait)
+		processBatchReliably(ctx, logger, registry, writer, reader, route, batch, cfg.OperationTimeout, cfg.RetryDelay)
+	}
+}
+
+func collectBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
+	if maxSize <= 1 {
+		return []kafka.Message{first}
+	}
+	batch := []kafka.Message{first}
+	deadline := time.Now().Add(maxWait)
+	for len(batch) < maxSize {
+		remaining := time.Until(deadline)
+		if remaining <= 0 {
+			break
+		}
+		fetchCtx, cancel := context.WithTimeout(ctx, remaining)
+		message, err := fetcher.FetchMessage(fetchCtx)
+		cancel()
+		if err != nil {
+			break
+		}
+		batch = append(batch, message)
+	}
+	return batch
+}
+
+func processBatchReliably(ctx context.Context, logger interface {
+	Error(string, ...any)
+	Warn(string, ...any)
+}, registry *metrics.Registry, writer kafkaBatchWriter, committer kafkaMessageCommitter, route projectionRoute, messages []kafka.Message, operationTimeout time.Duration, retryDelay time.Duration) {
+	labels := metrics.Labels{"protocol": string(route.Protocol)}
+	defer registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, 0)
+	for len(messages) > 0 {
+		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
+		outputs, err := projectBatch(operationCtx, logger, registry, writer, route, messages)
+		cancel()
+		if err != nil {
+			registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages)))
+			registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "write_error"})
+			if !waitForRetry(ctx, retryDelay) {
+				return
+			}
+			continue
+		}
+		commitCtx, commitCancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
+		err = committer.CommitMessages(commitCtx, messages...)
+		commitCancel()
+		if err == nil {
+			for _, message := range messages {
+				recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok")
+			}
+			_ = outputs
+			return
+		}
+		for _, message := range messages {
+			recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error")
+		}
+		registry.SetGauge("vehicle_fields_projector_retry_pending_messages", labels, float64(len(messages)))
+		registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(route.Protocol), "reason": "commit_error"})
+		logger.Error("kafka source offset commit failed", "topic", route.RawTopic, "messages", len(messages), "error", err)
+		if !retryCommit(ctx, logger, registry, committer, messages, route.Protocol, operationTimeout, retryDelay) {
+			return
+		}
+		return
+	}
+}
+
+func projectBatch(ctx context.Context, logger interface {
+	Error(string, ...any)
+	Warn(string, ...any)
+}, registry *metrics.Registry, writer kafkaBatchWriter, route projectionRoute, messages []kafka.Message) (int, error) {
+	if len(messages) == 0 {
+		return 0, nil
+	}
+	outputs := make([]kafka.Message, 0, len(messages))
+	setBatchPending(registry, route.Protocol, len(messages), 0)
+	defer setBatchPending(registry, route.Protocol, -len(messages), -len(outputs))
+	for _, message := range messages {
+		recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "received")
+		recordKafkaLag(registry, message)
+		var raw envelope.FrameEnvelope
+		if err := json.Unmarshal(message.Value, &raw); err != nil {
+			recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, "invalid_json")
+			logger.Warn("skip invalid raw envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
+			continue
+		}
+		if status, err := topics.ValidateRawEnvelope(route.RawTopic, raw); err != nil {
+			recordMessageMetric(registry, "vehicle_fields_projector_kafka_messages_total", message.Topic, status)
+			logger.Warn("skip mismatched raw envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", raw.Protocol, "event_id", raw.StableEventID(), "error", err)
+			continue
+		}
+		fields, ok := realtime.BuildFieldsEnvelope(raw)
+		if !ok {
+			status := "skipped_missing_fields"
+			if !envelope.IsRealtimeTelemetryFrame(raw) {
+				status = "skipped_non_realtime"
+			}
+			recordProjectionMetric(registry, route.Protocol, status, 0)
+			continue
+		}
+		if status, err := topics.ValidateFieldsEnvelope(route.FieldsTopic, fields); err != nil {
+			recordProjectionMetric(registry, route.Protocol, status, len(fields.Fields))
+			logger.Warn("skip invalid projected fields envelope", "topic", route.FieldsTopic, "protocol", fields.Protocol, "event_id", fields.StableEventID(), "error", err)
+			continue
+		}
+		payload, err := fields.MarshalJSONBytes()
+		if err != nil {
+			recordProjectionMetric(registry, route.Protocol, "marshal_error", len(fields.Fields))
+			logger.Warn("skip fields envelope marshal error", "topic", route.FieldsTopic, "event_id", fields.StableEventID(), "error", err)
+			continue
+		}
+		outputs = append(outputs, kafka.Message{
+			Topic: route.FieldsTopic,
+			Key:   fields.KafkaKey(),
+			Value: payload,
+			Time:  message.Time,
+		})
+		recordProjectionMetric(registry, route.Protocol, "projected", len(fields.Fields))
+	}
+	setBatchPending(registry, route.Protocol, 0, len(outputs))
+	defer setBatchPending(registry, route.Protocol, 0, -len(outputs))
+	if len(outputs) == 0 {
+		return 0, nil
+	}
+	started := time.Now()
+	err := writer.WriteMessages(ctx, outputs...)
+	status := "ok"
+	if err != nil {
+		status = "error"
+	}
+	recordWriteDuration(registry, route.FieldsTopic, status, time.Since(started))
+	for range outputs {
+		recordMessageMetric(registry, "vehicle_fields_projector_kafka_writes_total", route.FieldsTopic, status)
+	}
+	if err != nil {
+		logger.Error("fields kafka write failed", "topic", route.FieldsTopic, "messages", len(outputs), "error", err)
+		return len(outputs), err
+	}
+	return len(outputs), nil
+}
+
+func retryCommit(ctx context.Context, logger interface {
+	Error(string, ...any)
+}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, protocol envelope.Protocol, operationTimeout, retryDelay time.Duration) bool {
+	for {
+		if !waitForRetry(ctx, retryDelay) {
+			return false
+		}
+		operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), operationTimeout)
+		err := committer.CommitMessages(operationCtx, messages...)
+		cancel()
+		if err == nil {
+			for _, message := range messages {
+				recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "ok")
+			}
+			return true
+		}
+		for _, message := range messages {
+			recordMessageMetric(registry, "vehicle_fields_projector_kafka_commits_total", message.Topic, "error")
+		}
+		registry.IncCounter("vehicle_fields_projector_batch_retries_total", metrics.Labels{"protocol": string(protocol), "reason": "commit_error"})
+		logger.Error("kafka source offset commit retry failed", "messages", len(messages), "error", err)
+	}
+}
+
+func waitForRetry(ctx context.Context, delay time.Duration) bool {
+	timer := time.NewTimer(delay)
+	defer timer.Stop()
+	select {
+	case <-ctx.Done():
+		return false
+	case <-timer.C:
+		return true
+	}
+}
+
+func pingKafka(ctx context.Context, brokers []string) error {
+	if len(brokers) == 0 {
+		return errors.New("kafka broker is required")
+	}
+	checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	defer cancel()
+	conn, err := kafka.DialContext(checkCtx, "tcp", brokers[0])
+	if err != nil {
+		return err
+	}
+	return conn.Close()
+}
+
+func recordConfigMetrics(registry *metrics.Registry, cfg config) {
+	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "workers_per_protocol"}, float64(cfg.WorkersPerProtocol))
+	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize))
+	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "batch_wait_ms"}, float64(cfg.BatchWait.Milliseconds()))
+	registry.SetGauge("vehicle_fields_projector_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationTimeout.Milliseconds()))
+}
+
+func recordMessageMetric(registry *metrics.Registry, name, topic, status string) {
+	if registry == nil {
+		return
+	}
+	labels := metrics.Labels{"topic": topic, "status": status}
+	registry.IncCounter(name, labels)
+	switch name {
+	case "vehicle_fields_projector_kafka_messages_total":
+		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_message_unix_seconds", labels)
+	case "vehicle_fields_projector_kafka_writes_total":
+		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_write_unix_seconds", labels)
+	case "vehicle_fields_projector_kafka_commits_total":
+		metrics.RecordLastActivity(registry, "vehicle_fields_projector_last_commit_unix_seconds", labels)
+	}
+}
+
+func recordProjectionMetric(registry *metrics.Registry, protocol envelope.Protocol, status string, fieldCount int) {
+	if registry == nil {
+		return
+	}
+	labels := metrics.Labels{"protocol": string(protocol), "status": status}
+	registry.IncCounter("vehicle_fields_projector_projections_total", labels)
+	if fieldCount > 0 {
+		registry.SetGauge("vehicle_fields_projector_field_count", labels, float64(fieldCount))
+	}
+}
+
+func recordKafkaLag(registry *metrics.Registry, message kafka.Message) {
+	if registry == nil {
+		return
+	}
+	lag := message.HighWaterMark - message.Offset - 1
+	if lag < 0 {
+		lag = 0
+	}
+	registry.SetGauge("vehicle_fields_projector_kafka_lag", metrics.Labels{
+		"topic": message.Topic, "partition": strconv.Itoa(message.Partition),
+	}, float64(lag))
+}
+
+var projectorPendingMu sync.Mutex
+var projectorPendingMessages = map[envelope.Protocol]int{}
+var projectorPendingFields = map[envelope.Protocol]int{}
+
+func setBatchPending(registry *metrics.Registry, protocol envelope.Protocol, messagesDelta, fieldsDelta int) {
+	if registry == nil {
+		return
+	}
+	projectorPendingMu.Lock()
+	projectorPendingMessages[protocol] += messagesDelta
+	projectorPendingFields[protocol] += fieldsDelta
+	messages := projectorPendingMessages[protocol]
+	fields := projectorPendingFields[protocol]
+	projectorPendingMu.Unlock()
+	labels := metrics.Labels{"protocol": string(protocol)}
+	registry.SetGauge("vehicle_fields_projector_batch_pending_messages", labels, float64(messages))
+	registry.SetGauge("vehicle_fields_projector_batch_pending_fields", labels, float64(fields))
+}
+
+var projectorWriteBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
+
+func recordWriteDuration(registry *metrics.Registry, topic, status string, elapsed time.Duration) {
+	if registry == nil {
+		return
+	}
+	registry.ObserveHistogram("vehicle_fields_projector_write_duration_ms_histogram", metrics.Labels{
+		"topic": topic, "status": status,
+	}, projectorWriteBucketsMS, float64(elapsed.Milliseconds()))
+}
+
+func env(key, fallback string) string {
+	value := strings.TrimSpace(os.Getenv(key))
+	if value == "" {
+		return fallback
+	}
+	return value
+}
+
+func envInt(key string, fallback int) int {
+	value := strings.TrimSpace(os.Getenv(key))
+	if value == "" {
+		return fallback
+	}
+	parsed, err := strconv.Atoi(value)
+	if err != nil {
+		return fallback
+	}
+	return parsed
+}
+
+func splitCSV(value string) []string {
+	var out []string
+	for _, item := range strings.Split(value, ",") {
+		item = strings.TrimSpace(item)
+		if item != "" {
+			out = append(out, item)
+		}
+	}
+	return out
+}
--- a/go/vehicle-gateway/cmd/fields-projector/main_test.go
+++ b/go/vehicle-gateway/cmd/fields-projector/main_test.go
@@ -0,0 +1,242 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/topics"
+)
+
+func TestLoadConfigCreatesProtocolIsolatedConsumerGroups(t *testing.T) {
+	t.Setenv("FIELDS_PROJECTOR_GROUP_PREFIX", "projector-test")
+	t.Setenv("KAFKA_START_OFFSET", "first")
+	cfg := loadConfig()
+	if cfg.StartOffset != kafka.FirstOffset || cfg.StartOffsetName != "first" {
+		t.Fatalf("start offset = %d/%q", cfg.StartOffset, cfg.StartOffsetName)
+	}
+	wantGroups := map[envelope.Protocol]string{
+		envelope.ProtocolGB32960:    "projector-test-gb32960",
+		envelope.ProtocolJT808:      "projector-test-jt808",
+		envelope.ProtocolYutongMQTT: "projector-test-yutong-mqtt",
+	}
+	for _, route := range cfg.Routes {
+		if route.GroupID != wantGroups[route.Protocol] {
+			t.Fatalf("group for %s = %q", route.Protocol, route.GroupID)
+		}
+	}
+	if err := cfg.Validate(); err != nil {
+		t.Fatalf("Validate() error = %v", err)
+	}
+}
+
+func TestProjectBatchUsesPrecomputedFieldsAndPreservesSourceMetadata(t *testing.T) {
+	raw := projectorRawEnvelope()
+	payload, err := raw.MarshalJSONBytes()
+	if err != nil {
+		t.Fatalf("marshal raw: %v", err)
+	}
+	writer := &recordingProjectorWriter{}
+	registry := metrics.NewRegistry()
+	route := jt808ProjectionRoute()
+	count, err := projectBatch(context.Background(), discardProjectorLogger{}, registry, writer, route, []kafka.Message{{
+		Topic: route.RawTopic, Key: raw.KafkaKey(), Value: payload, Partition: 2, Offset: 10, HighWaterMark: 11,
+	}})
+	if err != nil {
+		t.Fatalf("projectBatch() error = %v", err)
+	}
+	if count != 1 || writer.callCount() != 1 || len(writer.messages) != 1 {
+		t.Fatalf("projected=%d calls=%d messages=%d", count, writer.callCount(), len(writer.messages))
+	}
+	var fields envelope.FrameEnvelope
+	if err := json.Unmarshal(writer.messages[0].Value, &fields); err != nil {
+		t.Fatalf("decode fields: %v", err)
+	}
+	if fields.EventKind != envelope.EventKindFields || fields.SourceEventID != raw.EventID || fields.EventID != raw.EventID+":fields" {
+		t.Fatalf("fields identity = %#v", fields)
+	}
+	if fields.SourceCode != raw.SourceCode || fields.SourceKind != raw.SourceKind || fields.SourceEndpoint != raw.SourceEndpoint {
+		t.Fatalf("source metadata not preserved: %#v", fields)
+	}
+	if got := fields.Fields["jt808.location.total_mileage_km"]; got != 1234.5 {
+		t.Fatalf("total mileage = %#v", got)
+	}
+	if len(fields.Parsed) != 0 || len(fields.ParsedFields) != 0 {
+		t.Fatalf("fields projection must not duplicate raw payload: parsed=%v parsed_fields=%v", fields.Parsed, fields.ParsedFields)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_fields_projector_projections_total{protocol="JT808",status="projected"} 1`,
+		`vehicle_fields_projector_kafka_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
+		`vehicle_fields_projector_kafka_lag{partition="2",topic="vehicle.raw.go.jt808.v1"} 0`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("metric missing %q:\n%s", want, text)
+		}
+	}
+}
+
+func TestProcessBatchReliablySkipsNonRealtimeAndInvalidWithoutWriting(t *testing.T) {
+	route := jt808ProjectionRoute()
+	nonRealtime := projectorRawEnvelope()
+	nonRealtime.MessageID = "0x0002"
+	nonRealtime.ParsedFields = map[string]any{"jt808.header.message_id": "0x0002"}
+	payload, _ := nonRealtime.MarshalJSONBytes()
+	messages := []kafka.Message{
+		{Topic: route.RawTopic, Value: []byte("{bad"), Partition: 0, Offset: 1, HighWaterMark: 3},
+		{Topic: route.RawTopic, Value: payload, Partition: 0, Offset: 2, HighWaterMark: 3},
+	}
+	writer := &recordingProjectorWriter{}
+	committer := &recordingProjectorCommitter{}
+	registry := metrics.NewRegistry()
+	processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
+	if writer.callCount() != 0 {
+		t.Fatalf("writer calls = %d, want 0", writer.callCount())
+	}
+	if committer.callCount() != 1 || committer.messageCount != 2 {
+		t.Fatalf("commit calls/messages = %d/%d", committer.callCount(), committer.messageCount)
+	}
+	text := registry.Render()
+	if !strings.Contains(text, `vehicle_fields_projector_kafka_messages_total{status="invalid_json",topic="vehicle.raw.go.jt808.v1"} 1`) ||
+		!strings.Contains(text, `vehicle_fields_projector_projections_total{protocol="JT808",status="skipped_non_realtime"} 1`) {
+		t.Fatalf("skip metrics missing:\n%s", text)
+	}
+}
+
+func TestProcessBatchReliablyRetriesWriteBeforeCommitting(t *testing.T) {
+	route := jt808ProjectionRoute()
+	payload, _ := projectorRawEnvelope().MarshalJSONBytes()
+	messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
+	writer := &recordingProjectorWriter{errors: []error{errors.New("kafka unavailable"), nil}}
+	committer := &recordingProjectorCommitter{}
+	registry := metrics.NewRegistry()
+	processBatchReliably(context.Background(), discardProjectorLogger{}, registry, writer, committer, route, messages, time.Second, time.Millisecond)
+	if writer.callCount() != 2 {
+		t.Fatalf("writer calls = %d, want 2", writer.callCount())
+	}
+	if committer.callCount() != 1 {
+		t.Fatalf("commit calls = %d, want 1 after successful write", committer.callCount())
+	}
+	if !strings.Contains(registry.Render(), `vehicle_fields_projector_batch_retries_total{protocol="JT808",reason="write_error"} 1`) {
+		t.Fatalf("write retry metric missing:\n%s", registry.Render())
+	}
+}
+
+func TestProcessBatchReliablyRetriesOnlyCommitAfterSuccessfulWrite(t *testing.T) {
+	route := jt808ProjectionRoute()
+	payload, _ := projectorRawEnvelope().MarshalJSONBytes()
+	messages := []kafka.Message{{Topic: route.RawTopic, Value: payload, Partition: 1, Offset: 7, HighWaterMark: 8}}
+	writer := &recordingProjectorWriter{}
+	committer := &recordingProjectorCommitter{errors: []error{errors.New("commit timeout"), nil}}
+	processBatchReliably(context.Background(), discardProjectorLogger{}, metrics.NewRegistry(), writer, committer, route, messages, time.Second, time.Millisecond)
+	if writer.callCount() != 1 {
+		t.Fatalf("writer calls = %d, want 1", writer.callCount())
+	}
+	if committer.callCount() != 2 {
+		t.Fatalf("commit calls = %d, want 2", committer.callCount())
+	}
+}
+
+func TestProjectBatchWriteFailureDoesNotCommitByItself(t *testing.T) {
+	route := jt808ProjectionRoute()
+	payload, _ := projectorRawEnvelope().MarshalJSONBytes()
+	writer := &recordingProjectorWriter{errors: []error{errors.New("write failed")}}
+	count, err := projectBatch(context.Background(), discardProjectorLogger{}, nil, writer, route, []kafka.Message{{Topic: route.RawTopic, Value: payload}})
+	if err == nil || count != 1 {
+		t.Fatalf("projectBatch() count/error = %d/%v", count, err)
+	}
+}
+
+func projectorRawEnvelope() envelope.FrameEnvelope {
+	return envelope.FrameEnvelope{
+		EventID:        "raw-event-1",
+		EventKind:      envelope.EventKindRaw,
+		Protocol:       envelope.ProtocolJT808,
+		MessageID:      "0x0200",
+		VIN:            "VIN001",
+		Phone:          "13307795425",
+		SourceEndpoint: "115.231.168.135:43625",
+		SourceCode:     "g7s",
+		SourceKind:     "PLATFORM",
+		PlatformName:   "G7s",
+		EventTimeMS:    1783960000000,
+		ReceivedAtMS:   1783960000010,
+		ParsedFields: map[string]any{
+			"jt808.location.latitude":         30.1,
+			"jt808.location.longitude":        121.2,
+			"jt808.location.total_mileage_km": 1234.5,
+		},
+		ParseStatus: envelope.ParseOK,
+	}
+}
+
+func jt808ProjectionRoute() projectionRoute {
+	return projectionRoute{
+		Protocol: envelope.ProtocolJT808, RawTopic: topics.RawJT808, FieldsTopic: topics.FieldsJT808, GroupID: "projector-jt808",
+	}
+}
+
+type recordingProjectorWriter struct {
+	mu       sync.Mutex
+	errors   []error
+	calls    int
+	messages []kafka.Message
+}
+
+func (w *recordingProjectorWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.calls++
+	w.messages = append(w.messages, messages...)
+	if len(w.errors) == 0 {
+		return nil
+	}
+	err := w.errors[0]
+	w.errors = w.errors[1:]
+	return err
+}
+
+func (w *recordingProjectorWriter) callCount() int {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	return w.calls
+}
+
+type recordingProjectorCommitter struct {
+	mu           sync.Mutex
+	errors       []error
+	calls        int
+	messageCount int
+}
+
+func (c *recordingProjectorCommitter) CommitMessages(_ context.Context, messages ...kafka.Message) error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.calls++
+	c.messageCount += len(messages)
+	if len(c.errors) == 0 {
+		return nil
+	}
+	err := c.errors[0]
+	c.errors = c.errors[1:]
+	return err
+}
+
+func (c *recordingProjectorCommitter) callCount() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.calls
+}
+
+type discardProjectorLogger struct{}
+
+func (discardProjectorLogger) Error(string, ...any) {}
+func (discardProjectorLogger) Warn(string, ...any)  {}