feat: scaffold go vehicle gateway core

2026-07-01 21:43:16 +08:00
parent d148a4e3af
commit 1fdbc29e27
12 changed files with 1107 additions and 0 deletions
--- a/go/vehicle-gateway/cmd/gateway/main.go
+++ b/go/vehicle-gateway/cmd/gateway/main.go
@@ -0,0 +1,8 @@
+package main
+
+import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
+
+func main() {
+	logger := observability.NewLogger("vehicle-gateway")
+	logger.Info("vehicle gateway scaffold started")
+}
--- a/go/vehicle-gateway/go.mod
+++ b/go/vehicle-gateway/go.mod
@@ -0,0 +1,9 @@
+module lingniu-vehicle-ingest/go/vehicle-gateway
+
+go 1.26
+
+require (
+	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	github.com/segmentio/kafka-go v0.4.49 // indirect
+)
--- a/go/vehicle-gateway/go.sum
+++ b/go/vehicle-gateway/go.sum
@@ -0,0 +1,6 @@
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
+github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
--- a/go/vehicle-gateway/internal/envelope/envelope.go
+++ b/go/vehicle-gateway/internal/envelope/envelope.go
@@ -0,0 +1,95 @@
+package envelope
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+type Protocol string
+
+const (
+	ProtocolGB32960    Protocol = "GB32960"
+	ProtocolJT808      Protocol = "JT808"
+	ProtocolYutongMQTT Protocol = "YUTONG_MQTT"
+)
+
+type ParseStatus string
+
+const (
+	ParseOK       ParseStatus = "OK"
+	ParsePartial  ParseStatus = "PARTIAL"
+	ParseBadFrame ParseStatus = "BAD_FRAME"
+)
+
+const (
+	FieldSpeedKMH       = "speed_kmh"
+	FieldTotalMileageKM = "total_mileage_km"
+	FieldLongitude      = "longitude"
+	FieldLatitude       = "latitude"
+	FieldSOCPercent     = "soc_percent"
+)
+
+type FrameEnvelope struct {
+	EventID        string         `json:"event_id"`
+	TraceID        string         `json:"trace_id"`
+	Protocol       Protocol       `json:"protocol"`
+	MessageID      string         `json:"message_id"`
+	Sequence       uint16         `json:"sequence"`
+	VIN            string         `json:"vin,omitempty"`
+	VehicleKeyHint string         `json:"vehicle_key,omitempty"`
+	Phone          string         `json:"phone,omitempty"`
+	DeviceID       string         `json:"device_id,omitempty"`
+	Plate          string         `json:"plate,omitempty"`
+	SourceEndpoint string         `json:"source_endpoint,omitempty"`
+	EventTimeMS    int64          `json:"event_time_ms"`
+	ReceivedAtMS   int64          `json:"received_at_ms"`
+	RawHex         string         `json:"raw_hex,omitempty"`
+	RawText        string         `json:"raw_text,omitempty"`
+	Parsed         map[string]any `json:"parsed,omitempty"`
+	Fields         map[string]any `json:"fields,omitempty"`
+	ParseStatus    ParseStatus    `json:"parse_status"`
+	ParseError     string         `json:"parse_error,omitempty"`
+}
+
+func (e FrameEnvelope) VehicleKey() string {
+	if key := strings.TrimSpace(e.VIN); key != "" {
+		return key
+	}
+	if key := strings.TrimSpace(e.VehicleKeyHint); key != "" {
+		return key
+	}
+	if key := strings.TrimSpace(e.Phone); key != "" {
+		return string(e.Protocol) + ":" + key
+	}
+	if key := strings.TrimSpace(e.DeviceID); key != "" {
+		return string(e.Protocol) + ":" + key
+	}
+	return string(e.Protocol) + ":unknown"
+}
+
+func (e FrameEnvelope) StableEventID() string {
+	if strings.TrimSpace(e.EventID) != "" {
+		return e.EventID
+	}
+	input := fmt.Sprintf("%s|%s|%s|%d|%d|%s",
+		e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex)
+	sum := sha256.Sum256([]byte(input))
+	return hex.EncodeToString(sum[:16])
+}
+
+func (e FrameEnvelope) KafkaKey() []byte {
+	return []byte(e.VehicleKey())
+}
+
+func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) {
+	if e.EventID == "" {
+		e.EventID = e.StableEventID()
+	}
+	if e.ParseStatus == "" {
+		e.ParseStatus = ParseOK
+	}
+	return json.Marshal(e)
+}
--- a/go/vehicle-gateway/internal/envelope/envelope_test.go
+++ b/go/vehicle-gateway/internal/envelope/envelope_test.go
@@ -0,0 +1,55 @@
+package envelope
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) {
+	e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"}
+	if got := e.VehicleKey(); got != "LNBVIN00000000001" {
+		t.Fatalf("VehicleKey() = %q", got)
+	}
+}
+
+func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) {
+	e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"}
+	if got := e.VehicleKey(); got != "JT808:013307795425" {
+		t.Fatalf("VehicleKey() = %q", got)
+	}
+}
+
+func TestFrameEnvelopeEventIDStable(t *testing.T) {
+	e := FrameEnvelope{
+		Protocol:     ProtocolJT808,
+		MessageID:    "0x0200",
+		Phone:        "013307795425",
+		Sequence:     1,
+		EventTimeMS:  1782745114000,
+		ReceivedAtMS: 1782745114999,
+		RawHex:       "7e02000000ff7e",
+	}
+	a := e.StableEventID()
+	b := e.StableEventID()
+	if a == "" || a != b {
+		t.Fatalf("event id must be non-empty and stable: %q %q", a, b)
+	}
+}
+
+func TestFrameEnvelopeMarshalDefaults(t *testing.T) {
+	e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"}
+	payload, err := e.MarshalJSONBytes()
+	if err != nil {
+		t.Fatalf("MarshalJSONBytes() error = %v", err)
+	}
+	var decoded FrameEnvelope
+	if err := json.Unmarshal(payload, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+	if decoded.EventID == "" {
+		t.Fatal("event id should be filled")
+	}
+	if decoded.ParseStatus != ParseOK {
+		t.Fatalf("parse status = %q", decoded.ParseStatus)
+	}
+}
--- a/go/vehicle-gateway/internal/eventbus/kafka_sink.go
+++ b/go/vehicle-gateway/internal/eventbus/kafka_sink.go
@@ -0,0 +1,94 @@
+package eventbus
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/segmentio/kafka-go"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+type Sink interface {
+	PublishRaw(context.Context, envelope.FrameEnvelope) error
+	PublishUnified(context.Context, envelope.FrameEnvelope) error
+	Close() error
+}
+
+type KafkaSink struct {
+	writer       kafkaWriter
+	rawTopics    map[envelope.Protocol]string
+	unifiedTopic string
+}
+
+type kafkaWriter interface {
+	WriteMessages(context.Context, ...kafka.Message) error
+	Close() error
+}
+
+type KafkaConfig struct {
+	Brokers      []string
+	RawTopics    map[envelope.Protocol]string
+	UnifiedTopic string
+}
+
+func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
+	if len(cfg.Brokers) == 0 {
+		return nil, errors.New("kafka brokers are required")
+	}
+	return newKafkaSinkWithWriter(&kafka.Writer{
+		Addr:                   kafka.TCP(cfg.Brokers...),
+		Balancer:               &kafka.Hash{},
+		AllowAutoTopicCreation: false,
+	}, cfg), nil
+}
+
+func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
+	rawTopics := map[envelope.Protocol]string{
+		envelope.ProtocolGB32960:    "vehicle.raw.gb32960.v1",
+		envelope.ProtocolJT808:      "vehicle.raw.jt808.v1",
+		envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1",
+	}
+	for protocol, topic := range cfg.RawTopics {
+		if topic != "" {
+			rawTopics[protocol] = topic
+		}
+	}
+	unifiedTopic := cfg.UnifiedTopic
+	if unifiedTopic == "" {
+		unifiedTopic = "vehicle.event.unified.v1"
+	}
+	return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic}
+}
+
+func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
+	topic, ok := s.rawTopics[env.Protocol]
+	if !ok || topic == "" {
+		return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol)
+	}
+	return s.publish(ctx, topic, env)
+}
+
+func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
+	return s.publish(ctx, s.unifiedTopic, env)
+}
+
+func (s *KafkaSink) Close() error {
+	if s == nil || s.writer == nil {
+		return nil
+	}
+	return s.writer.Close()
+}
+
+func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
+	payload, err := env.MarshalJSONBytes()
+	if err != nil {
+		return err
+	}
+	return s.writer.WriteMessages(ctx, kafka.Message{
+		Topic: topic,
+		Key:   env.KafkaKey(),
+		Value: payload,
+	})
+}
--- a/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
+++ b/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
@@ -0,0 +1,77 @@
+package eventbus
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	"github.com/segmentio/kafka-go"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+func TestKafkaSinkRoutesRawTopics(t *testing.T) {
+	writer := &recordingWriter{}
+	sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
+
+	env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"}
+	if err := sink.PublishRaw(context.Background(), env); err != nil {
+		t.Fatalf("PublishRaw() error = %v", err)
+	}
+	if len(writer.messages) != 1 {
+		t.Fatalf("messages = %d", len(writer.messages))
+	}
+	msg := writer.messages[0]
+	if msg.Topic != "vehicle.raw.jt808.v1" {
+		t.Fatalf("topic = %q", msg.Topic)
+	}
+	if string(msg.Key) != "JT808:013307795425" {
+		t.Fatalf("key = %q", string(msg.Key))
+	}
+	var decoded envelope.FrameEnvelope
+	if err := json.Unmarshal(msg.Value, &decoded); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+	if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK {
+		t.Fatalf("unexpected payload defaults: %#v", decoded)
+	}
+}
+
+func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) {
+	writer := &recordingWriter{}
+	sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"})
+
+	env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"}
+	if err := sink.PublishUnified(context.Background(), env); err != nil {
+		t.Fatalf("PublishUnified() error = %v", err)
+	}
+	if writer.messages[0].Topic != "custom.unified" {
+		t.Fatalf("topic = %q", writer.messages[0].Topic)
+	}
+	if string(writer.messages[0].Key) != "LNBVIN00000000001" {
+		t.Fatalf("key = %q", string(writer.messages[0].Key))
+	}
+}
+
+func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) {
+	writer := &recordingWriter{}
+	sink := newKafkaSinkWithWriter(writer, KafkaConfig{})
+
+	err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")})
+	if err == nil {
+		t.Fatal("expected unknown protocol error")
+	}
+}
+
+type recordingWriter struct {
+	messages []kafka.Message
+}
+
+func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error {
+	w.messages = append(w.messages, messages...)
+	return nil
+}
+
+func (w *recordingWriter) Close() error {
+	return nil
+}
--- a/go/vehicle-gateway/internal/observability/logger.go
+++ b/go/vehicle-gateway/internal/observability/logger.go
@@ -0,0 +1,13 @@
+package observability
+
+import (
+	"log/slog"
+	"os"
+)
+
+// NewLogger returns a JSON logger with a stable service field so ECS logs from
+// different Go processes can be filtered without relying on container names.
+func NewLogger(service string) *slog.Logger {
+	handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true})
+	return slog.New(handler).With("service", service)
+}
--- a/go/vehicle-gateway/internal/protocol/gb32960/parser.go
+++ b/go/vehicle-gateway/internal/protocol/gb32960/parser.go
@@ -0,0 +1,237 @@
+package gb32960
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+var (
+	ErrFrameTooShort  = errors.New("gb32960 frame too short")
+	ErrBadStartSymbol = errors.New("gb32960 bad start symbol")
+	ErrBodyLength     = errors.New("gb32960 body length mismatch")
+	ErrBCC            = errors.New("gb32960 bcc mismatch")
+)
+
+const (
+	minFrameLen = 25
+	headerLen   = 24
+)
+
+// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field.
+// Returned frames include the start symbols and trailing BCC byte.
+func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
+	offset := 0
+	for {
+		start := indexStart(stream[offset:])
+		if start < 0 {
+			return frames, nil, nil
+		}
+		offset += start
+		remaining := stream[offset:]
+		if len(remaining) < headerLen {
+			return frames, append([]byte(nil), remaining...), nil
+		}
+		bodyLen := int(binary.BigEndian.Uint16(remaining[22:24]))
+		frameLen := headerLen + bodyLen + 1
+		if len(remaining) < frameLen {
+			return frames, append([]byte(nil), remaining...), nil
+		}
+		frames = append(frames, append([]byte(nil), remaining[:frameLen]...))
+		offset += frameLen
+		if offset >= len(stream) {
+			return frames, nil, nil
+		}
+	}
+}
+
+func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
+	if len(raw) < minFrameLen {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
+	}
+	version, ok := version(raw[0], raw[1])
+	if !ok {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2]))
+	}
+	bodyLen := int(binary.BigEndian.Uint16(raw[22:24]))
+	if len(raw) != headerLen+bodyLen+1 {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1)
+	}
+	if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want)
+	}
+
+	command := raw[2]
+	responseFlag := raw[3]
+	vin := strings.TrimRight(string(raw[4:21]), "\x00 ")
+	body := raw[headerLen : headerLen+bodyLen]
+	parsed := map[string]any{
+		"header": map[string]any{
+			"version":       version,
+			"command":       fmt.Sprintf("0x%02X", command),
+			"response_flag": fmt.Sprintf("0x%02X", responseFlag),
+			"vin":           vin,
+			"encrypt":       raw[21],
+			"body_length":   bodyLen,
+		},
+	}
+	fields := map[string]any{}
+	eventTimeMS := receivedAtMS
+
+	if command == 0x02 || command == 0x03 {
+		eventTime, units := parseDataBody(version, body, fields)
+		if !eventTime.IsZero() {
+			eventTimeMS = eventTime.UnixMilli()
+			fields["device_time"] = eventTime.Format(time.RFC3339)
+			parsed["device_time"] = eventTime.Format(time.RFC3339)
+		}
+		parsed["data_units"] = units
+	}
+
+	env := envelope.FrameEnvelope{
+		Protocol:       envelope.ProtocolGB32960,
+		MessageID:      fmt.Sprintf("0x%02X", command),
+		VIN:            vin,
+		SourceEndpoint: sourceEndpoint,
+		EventTimeMS:    eventTimeMS,
+		ReceivedAtMS:   receivedAtMS,
+		RawHex:         strings.ToUpper(hex.EncodeToString(raw)),
+		Parsed:         parsed,
+		Fields:         fields,
+		ParseStatus:    envelope.ParseOK,
+	}
+	env.EventID = env.StableEventID()
+	return env, nil
+}
+
+func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) {
+	if len(body) < 6 {
+		return time.Time{}, nil
+	}
+	eventTime := parseBCDTime(body[:6])
+	cursor := 6
+	var units []map[string]any
+	for cursor < len(body) {
+		unitType := body[cursor]
+		cursor++
+		switch unitType {
+		case 0x01:
+			size := 20
+			if version == "V2025" {
+				size = 18
+			}
+			if len(body[cursor:]) < size {
+				units = append(units, map[string]any{"type": "0x01", "error": "truncated"})
+				return eventTime, units
+			}
+			unit := parseVehicleData(body[cursor : cursor+size])
+			units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit})
+			fields["vehicle_status"] = unit["vehicle_status"]
+			fields["charge_status"] = unit["charge_status"]
+			fields["running_mode"] = unit["running_mode"]
+			fields[envelope.FieldSpeedKMH] = unit["speed_kmh"]
+			fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"]
+			fields[envelope.FieldSOCPercent] = unit["soc_percent"]
+			cursor += size
+		case 0x05:
+			if len(body[cursor:]) < 9 {
+				units = append(units, map[string]any{"type": "0x05", "error": "truncated"})
+				return eventTime, units
+			}
+			unit := parsePositionData(body[cursor : cursor+9])
+			units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit})
+			fields["position_status"] = unit["position_status"]
+			fields[envelope.FieldLongitude] = unit["longitude"]
+			fields[envelope.FieldLatitude] = unit["latitude"]
+			cursor += 9
+		default:
+			units = append(units, map[string]any{
+				"type":      fmt.Sprintf("0x%02X", unitType),
+				"raw_tail":  strings.ToUpper(hex.EncodeToString(body[cursor:])),
+				"parse":     "unsupported_unit",
+				"byte_size": len(body) - cursor,
+			})
+			return eventTime, units
+		}
+	}
+	return eventTime, units
+}
+
+func parseVehicleData(data []byte) map[string]any {
+	return map[string]any{
+		"vehicle_status":   int(data[0]),
+		"charge_status":    int(data[1]),
+		"running_mode":     int(data[2]),
+		"speed_kmh":        float64(binary.BigEndian.Uint16(data[3:5])) / 10,
+		"total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10,
+		"total_voltage_v":  float64(binary.BigEndian.Uint16(data[9:11])) / 10,
+		"total_current_a":  float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000,
+		"soc_percent":      int(data[13]),
+		"dc_dc_status":     int(data[14]),
+		"gear":             int(data[15]),
+		"insulation_kohm":  binary.BigEndian.Uint16(data[16:18]),
+		"accelerator_pct":  int(data[18]),
+		"brake_pct":        int(data[19]),
+	}
+}
+
+func parsePositionData(data []byte) map[string]any {
+	return map[string]any{
+		"position_status": int(data[0]),
+		"longitude":       float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000,
+		"latitude":        float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000,
+	}
+}
+
+func indexStart(data []byte) int {
+	for i := 0; i+1 < len(data); i++ {
+		if _, ok := version(data[i], data[i+1]); ok {
+			return i
+		}
+	}
+	return -1
+}
+
+func version(a byte, b byte) (string, bool) {
+	switch {
+	case a == '#' && b == '#':
+		return "V2016", true
+	case a == '$' && b == '$':
+		return "V2025", true
+	default:
+		return "", false
+	}
+}
+
+func parseBCDTime(data []byte) time.Time {
+	if len(data) != 6 {
+		return time.Time{}
+	}
+	year := 2000 + bcdByte(data[0])
+	month := time.Month(bcdByte(data[1]))
+	day := bcdByte(data[2])
+	hour := bcdByte(data[3])
+	minute := bcdByte(data[4])
+	second := bcdByte(data[5])
+	if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
+		return time.Time{}
+	}
+	return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
+}
+
+func bcdByte(value byte) int {
+	return int(value>>4)*10 + int(value&0x0f)
+}
+
+func bcc(data []byte) byte {
+	var out byte
+	for _, value := range data {
+		out ^= value
+	}
+	return out
+}
--- a/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
+++ b/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go
@@ -0,0 +1,106 @@
+package gb32960
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) {
+	first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57})
+	second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil)
+	stream := append([]byte{0x00, 0x01}, first...)
+	stream = append(stream, second[:len(second)-3]...)
+
+	frames, remainder, err := ExtractFrames(stream)
+	if err != nil {
+		t.Fatalf("ExtractFrames() error = %v", err)
+	}
+	if len(frames) != 1 {
+		t.Fatalf("expected one complete frame, got %d", len(frames))
+	}
+	if string(remainder) != string(second[:len(second)-3]) {
+		t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder))
+	}
+
+	env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960")
+	if err != nil {
+		t.Fatalf("ParseFrame() error = %v", err)
+	}
+	if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" {
+		t.Fatalf("unexpected envelope header: %#v", env)
+	}
+	if env.VIN != "LNBSCB3D4R1234567" {
+		t.Fatalf("unexpected vin: %q", env.VIN)
+	}
+}
+
+func TestParseFrameRejectsBadBCC(t *testing.T) {
+	frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
+	frame[len(frame)-1] ^= 0xff
+
+	if _, err := ParseFrame(frame, 1782745114999, ""); err == nil {
+		t.Fatal("expected bad bcc error")
+	}
+}
+
+func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) {
+	body := []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57}
+	body = append(body, 0x01)
+	body = append(body,
+		0x01,       // vehicle status
+		0x03,       // charge status
+		0x02,       // running mode
+		0x01, 0x2c, // speed: 30.0km/h
+		0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km
+		0x15, 0xe5, // total voltage: 560.5V
+		0x27, 0x10, // total current: 0A after -1000 offset
+		85, 0x01, 0x00,
+		0x27, 0x10,
+		0x00, 0x00)
+	body = append(body, 0x05)
+	body = append(body,
+		0x00,
+		0x07, 0x36, 0x50, 0x40, // longitude: 121.000000
+		0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000
+	frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body)
+
+	env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960")
+	if err != nil {
+		t.Fatalf("ParseFrame() error = %v", err)
+	}
+	assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0)
+	assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0)
+	assertFloatField(t, env, envelope.FieldLongitude, 121.0)
+	assertFloatField(t, env, envelope.FieldLatitude, 30.56)
+	if env.Fields[envelope.FieldSOCPercent] != 85 {
+		t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent])
+	}
+	if env.EventTimeMS == 1782745114999 {
+		t.Fatal("event time should come from GB32960 device timestamp")
+	}
+}
+
+func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
+	t.Helper()
+	got, ok := env.Fields[key].(float64)
+	if !ok {
+		t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
+	}
+	if got != want {
+		t.Fatalf("field %s = %v, want %v", key, got, want)
+	}
+}
+
+func buildFrame(command byte, response byte, vin string, body []byte) []byte {
+	frame := []byte{'#', '#', command, response}
+	vinBytes := []byte(vin)
+	if len(vinBytes) < 17 {
+		vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...)
+	}
+	frame = append(frame, vinBytes[:17]...)
+	frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body)))
+	frame = append(frame, body...)
+	return append(frame, bcc(frame[2:]))
+}
--- a/go/vehicle-gateway/internal/protocol/jt808/parser.go
+++ b/go/vehicle-gateway/internal/protocol/jt808/parser.go
@@ -0,0 +1,300 @@
+package jt808
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+var (
+	ErrFrameTooShort = errors.New("jt808 frame too short")
+	ErrChecksum      = errors.New("jt808 checksum mismatch")
+	ErrEscape        = errors.New("jt808 invalid escape sequence")
+)
+
+type Location struct {
+	AlarmFlag      uint32
+	StatusFlag     uint32
+	Latitude       float64
+	Longitude      float64
+	AltitudeM      uint16
+	SpeedKMH       float64
+	DirectionDeg   uint16
+	Time           time.Time
+	TotalMileageKM *float64
+	Additional     []AdditionalItem
+}
+
+type AdditionalItem struct {
+	ID       byte
+	Length   int
+	ValueHex string
+	Parsed   any
+}
+
+// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes
+// complete frame payloads. Returned frames exclude the leading and trailing 0x7e.
+func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) {
+	start := -1
+	for i, value := range stream {
+		if value != 0x7e {
+			continue
+		}
+		if start < 0 {
+			start = i
+			continue
+		}
+		if i == start+1 {
+			start = i
+			continue
+		}
+		frame, err := unescape(stream[start+1 : i])
+		if err != nil {
+			return nil, nil, err
+		}
+		frames = append(frames, frame)
+		start = i
+	}
+	if start >= 0 && start < len(stream)-1 {
+		return frames, append([]byte(nil), stream[start:]...), nil
+	}
+	return frames, nil, nil
+}
+
+func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) {
+	if len(raw) < 13 {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw))
+	}
+	if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want)
+	}
+
+	messageID := binary.BigEndian.Uint16(raw[0:2])
+	props := binary.BigEndian.Uint16(raw[2:4])
+	bodySize := props & 0x03ff
+	headerLen := 12
+	packageInfo := map[string]any(nil)
+	if props&0x2000 != 0 {
+		headerLen = 16
+		packageInfo = map[string]any{
+			"total": binary.BigEndian.Uint16(raw[12:14]),
+			"index": binary.BigEndian.Uint16(raw[14:16]),
+		}
+	}
+	if len(raw) < headerLen+int(bodySize)+1 {
+		return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw))
+	}
+
+	phoneBCD := raw[4:10]
+	phone := bcdString(phoneBCD)
+	body := raw[headerLen : headerLen+int(bodySize)]
+	parsed := map[string]any{
+		"header": map[string]any{
+			"message_id":       fmt.Sprintf("0x%04X", messageID),
+			"properties":       props,
+			"body_size":        bodySize,
+			"phone_bcd_hex":    strings.ToUpper(hex.EncodeToString(phoneBCD)),
+			"phone":            phone,
+			"phone_trim_zero":  strings.TrimLeft(phone, "0"),
+			"sequence":         binary.BigEndian.Uint16(raw[10:12]),
+			"subpackage":       props&0x2000 != 0,
+			"package_info":     packageInfo,
+			"encryption_flags": (props >> 10) & 0x07,
+		},
+	}
+	fields := map[string]any{}
+	eventTimeMS := receivedAtMS
+
+	if messageID == 0x0200 {
+		location, err := parseLocation(body)
+		if err != nil {
+			return envelope.FrameEnvelope{}, err
+		}
+		parsed["location"] = locationToMap(location)
+		fields[envelope.FieldLatitude] = location.Latitude
+		fields[envelope.FieldLongitude] = location.Longitude
+		fields[envelope.FieldSpeedKMH] = location.SpeedKMH
+		fields["altitude_m"] = location.AltitudeM
+		fields["direction_deg"] = location.DirectionDeg
+		fields["alarm_flag"] = location.AlarmFlag
+		fields["status_flag"] = location.StatusFlag
+		if !location.Time.IsZero() {
+			eventTimeMS = location.Time.UnixMilli()
+			fields["device_time"] = location.Time.Format(time.RFC3339)
+		}
+		if location.TotalMileageKM != nil {
+			fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
+		}
+	}
+
+	env := envelope.FrameEnvelope{
+		Protocol:       envelope.ProtocolJT808,
+		MessageID:      fmt.Sprintf("0x%04X", messageID),
+		Sequence:       binary.BigEndian.Uint16(raw[10:12]),
+		Phone:          phone,
+		SourceEndpoint: sourceEndpoint,
+		EventTimeMS:    eventTimeMS,
+		ReceivedAtMS:   receivedAtMS,
+		RawHex:         strings.ToUpper(hex.EncodeToString(raw)),
+		Parsed:         parsed,
+		Fields:         fields,
+		ParseStatus:    envelope.ParseOK,
+	}
+	env.EventID = env.StableEventID()
+	return env, nil
+}
+
+func parseLocation(body []byte) (Location, error) {
+	if len(body) < 28 {
+		return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body))
+	}
+	location := Location{
+		AlarmFlag:    binary.BigEndian.Uint32(body[0:4]),
+		StatusFlag:   binary.BigEndian.Uint32(body[4:8]),
+		Latitude:     float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000,
+		Longitude:    float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000,
+		AltitudeM:    binary.BigEndian.Uint16(body[16:18]),
+		SpeedKMH:     float64(binary.BigEndian.Uint16(body[18:20])) / 10,
+		DirectionDeg: binary.BigEndian.Uint16(body[20:22]),
+		Time:         parseBCDTime(body[22:28]),
+	}
+	location.Additional = parseAdditional(body[28:], &location)
+	return location, nil
+}
+
+func parseAdditional(data []byte, location *Location) []AdditionalItem {
+	var out []AdditionalItem
+	for len(data) >= 2 {
+		id := data[0]
+		size := int(data[1])
+		data = data[2:]
+		if len(data) < size {
+			out = append(out, AdditionalItem{
+				ID:       id,
+				Length:   size,
+				ValueHex: strings.ToUpper(hex.EncodeToString(data)),
+				Parsed:   "truncated",
+			})
+			return out
+		}
+		value := data[:size]
+		item := AdditionalItem{
+			ID:       id,
+			Length:   size,
+			ValueHex: strings.ToUpper(hex.EncodeToString(value)),
+		}
+		// JT/T 808-2011 table 27: id 0x01 is GPS total mileage, DWORD, 0.1 km.
+		if id == 0x01 && size == 4 {
+			mileage := float64(binary.BigEndian.Uint32(value)) / 10
+			location.TotalMileageKM = &mileage
+			item.Parsed = map[string]any{
+				"name":  envelope.FieldTotalMileageKM,
+				"value": mileage,
+				"unit":  "km",
+			}
+		}
+		out = append(out, item)
+		data = data[size:]
+	}
+	return out
+}
+
+func locationToMap(location Location) map[string]any {
+	out := map[string]any{
+		"alarm_flag":    location.AlarmFlag,
+		"status_flag":   location.StatusFlag,
+		"latitude":      location.Latitude,
+		"longitude":     location.Longitude,
+		"altitude_m":    location.AltitudeM,
+		"speed_kmh":     location.SpeedKMH,
+		"direction_deg": location.DirectionDeg,
+		"additional":    additionalToMaps(location.Additional),
+	}
+	if !location.Time.IsZero() {
+		out["device_time"] = location.Time.Format(time.RFC3339)
+	}
+	if location.TotalMileageKM != nil {
+		out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM
+	}
+	return out
+}
+
+func additionalToMaps(items []AdditionalItem) []map[string]any {
+	out := make([]map[string]any, 0, len(items))
+	for _, item := range items {
+		value := map[string]any{
+			"id":        fmt.Sprintf("0x%02X", item.ID),
+			"length":    item.Length,
+			"value_hex": item.ValueHex,
+		}
+		if item.Parsed != nil {
+			value["parsed"] = item.Parsed
+		}
+		out = append(out, value)
+	}
+	return out
+}
+
+func bcdString(data []byte) string {
+	out := make([]byte, 0, len(data)*2)
+	for _, value := range data {
+		out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f))
+	}
+	return string(out)
+}
+
+func parseBCDTime(data []byte) time.Time {
+	if len(data) != 6 {
+		return time.Time{}
+	}
+	year := 2000 + bcdByte(data[0])
+	month := time.Month(bcdByte(data[1]))
+	day := bcdByte(data[2])
+	hour := bcdByte(data[3])
+	minute := bcdByte(data[4])
+	second := bcdByte(data[5])
+	if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 {
+		return time.Time{}
+	}
+	return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600))
+}
+
+func bcdByte(value byte) int {
+	return int(value>>4)*10 + int(value&0x0f)
+}
+
+func unescape(data []byte) ([]byte, error) {
+	out := make([]byte, 0, len(data))
+	for i := 0; i < len(data); i++ {
+		if data[i] != 0x7d {
+			out = append(out, data[i])
+			continue
+		}
+		if i+1 >= len(data) {
+			return nil, ErrEscape
+		}
+		i++
+		switch data[i] {
+		case 0x01:
+			out = append(out, 0x7d)
+		case 0x02:
+			out = append(out, 0x7e)
+		default:
+			return nil, ErrEscape
+		}
+	}
+	return out, nil
+}
+
+func checksum(data []byte) byte {
+	var out byte
+	for _, value := range data {
+		out ^= value
+	}
+	return out
+}
--- a/go/vehicle-gateway/internal/protocol/jt808/parser_test.go
+++ b/go/vehicle-gateway/internal/protocol/jt808/parser_test.go
@@ -0,0 +1,107 @@
+package jt808
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+)
+
+const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
+
+func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) {
+	stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	frames, remainder, err := ExtractFrames(stream)
+	if err != nil {
+		t.Fatalf("ExtractFrames() error = %v", err)
+	}
+	if len(frames) != 1 {
+		t.Fatalf("expected one complete frame, got %d", len(frames))
+	}
+	if hex.EncodeToString(remainder) != "7e02000000" {
+		t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder))
+	}
+
+	env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625")
+	if err != nil {
+		t.Fatalf("ParseFrame() error = %v", err)
+	}
+	if env.Protocol != envelope.ProtocolJT808 {
+		t.Fatalf("protocol = %q", env.Protocol)
+	}
+	if env.MessageID != "0x0200" {
+		t.Fatalf("message id = %q", env.MessageID)
+	}
+	if env.Phone != "013307795425" {
+		t.Fatalf("phone = %q", env.Phone)
+	}
+	if env.Sequence != 1 {
+		t.Fatalf("sequence = %d", env.Sequence)
+	}
+	assertFloatField(t, env, envelope.FieldLatitude, 30.590151)
+	assertFloatField(t, env, envelope.FieldLongitude, 121.069881)
+	assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0)
+	assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2)
+	if env.Fields["direction_deg"] != uint16(79) {
+		t.Fatalf("direction = %#v", env.Fields["direction_deg"])
+	}
+
+	location, ok := env.Parsed["location"].(map[string]any)
+	if !ok {
+		t.Fatalf("parsed location missing: %#v", env.Parsed)
+	}
+	additional, ok := location["additional"].([]map[string]any)
+	if !ok || len(additional) == 0 {
+		t.Fatalf("additional fields missing: %#v", location["additional"])
+	}
+	if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" {
+		t.Fatalf("unexpected first additional item: %#v", additional[0])
+	}
+}
+
+func TestExtractFramesHandlesEscapedPayload(t *testing.T) {
+	payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d}
+	frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...)
+	frame = append(frame, 0x7e)
+
+	frames, remainder, err := ExtractFrames(frame)
+	if err != nil {
+		t.Fatalf("ExtractFrames() error = %v", err)
+	}
+	if len(frames) != 1 || len(remainder) != 0 {
+		t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder))
+	}
+	if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) {
+		t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0]))
+	}
+}
+
+func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) {
+	t.Helper()
+	got, ok := env.Fields[key].(float64)
+	if !ok {
+		t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key])
+	}
+	if got != want {
+		t.Fatalf("field %s = %v, want %v", key, got, want)
+	}
+}
+
+func escape(payload []byte) []byte {
+	var out []byte
+	for _, value := range payload {
+		switch value {
+		case 0x7e:
+			out = append(out, 0x7d, 0x02)
+		case 0x7d:
+			out = append(out, 0x7d, 0x01)
+		default:
+			out = append(out, value)
+		}
+	}
+	return out
+}