diff --git a/go/vehicle-gateway/cmd/gateway/main.go b/go/vehicle-gateway/cmd/gateway/main.go new file mode 100644 index 00000000..1a4e4620 --- /dev/null +++ b/go/vehicle-gateway/cmd/gateway/main.go @@ -0,0 +1,8 @@ +package main + +import "lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability" + +func main() { + logger := observability.NewLogger("vehicle-gateway") + logger.Info("vehicle gateway scaffold started") +} diff --git a/go/vehicle-gateway/go.mod b/go/vehicle-gateway/go.mod new file mode 100644 index 00000000..9e362c5e --- /dev/null +++ b/go/vehicle-gateway/go.mod @@ -0,0 +1,9 @@ +module lingniu-vehicle-ingest/go/vehicle-gateway + +go 1.26 + +require ( + github.com/klauspost/compress v1.15.9 // indirect + github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/segmentio/kafka-go v0.4.49 // indirect +) diff --git a/go/vehicle-gateway/go.sum b/go/vehicle-gateway/go.sum new file mode 100644 index 00000000..28ba83df --- /dev/null +++ b/go/vehicle-gateway/go.sum @@ -0,0 +1,6 @@ +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk= +github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E= diff --git a/go/vehicle-gateway/internal/envelope/envelope.go b/go/vehicle-gateway/internal/envelope/envelope.go new file mode 100644 index 00000000..bbd89b24 --- /dev/null +++ b/go/vehicle-gateway/internal/envelope/envelope.go @@ -0,0 +1,95 @@ +package envelope + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" +) + +type Protocol string + +const ( + ProtocolGB32960 Protocol = "GB32960" + ProtocolJT808 Protocol = "JT808" + ProtocolYutongMQTT Protocol = "YUTONG_MQTT" +) + +type ParseStatus string + +const ( + ParseOK ParseStatus = "OK" + ParsePartial ParseStatus = "PARTIAL" + ParseBadFrame ParseStatus = "BAD_FRAME" +) + +const ( + FieldSpeedKMH = "speed_kmh" + FieldTotalMileageKM = "total_mileage_km" + FieldLongitude = "longitude" + FieldLatitude = "latitude" + FieldSOCPercent = "soc_percent" +) + +type FrameEnvelope struct { + EventID string `json:"event_id"` + TraceID string `json:"trace_id"` + Protocol Protocol `json:"protocol"` + MessageID string `json:"message_id"` + Sequence uint16 `json:"sequence"` + VIN string `json:"vin,omitempty"` + VehicleKeyHint string `json:"vehicle_key,omitempty"` + Phone string `json:"phone,omitempty"` + DeviceID string `json:"device_id,omitempty"` + Plate string `json:"plate,omitempty"` + SourceEndpoint string `json:"source_endpoint,omitempty"` + EventTimeMS int64 `json:"event_time_ms"` + ReceivedAtMS int64 `json:"received_at_ms"` + RawHex string `json:"raw_hex,omitempty"` + RawText string `json:"raw_text,omitempty"` + Parsed map[string]any `json:"parsed,omitempty"` + Fields map[string]any `json:"fields,omitempty"` + ParseStatus ParseStatus `json:"parse_status"` + ParseError string `json:"parse_error,omitempty"` +} + +func (e FrameEnvelope) VehicleKey() string { + if key := strings.TrimSpace(e.VIN); key != "" { + return key + } + if key := strings.TrimSpace(e.VehicleKeyHint); key != "" { + return key + } + if key := strings.TrimSpace(e.Phone); key != "" { + return string(e.Protocol) + ":" + key + } + if key := strings.TrimSpace(e.DeviceID); key != "" { + return string(e.Protocol) + ":" + key + } + return string(e.Protocol) + ":unknown" +} + +func (e FrameEnvelope) StableEventID() string { + if strings.TrimSpace(e.EventID) != "" { + return e.EventID + } + input := fmt.Sprintf("%s|%s|%s|%d|%d|%s", + e.Protocol, e.MessageID, e.VehicleKey(), e.Sequence, e.EventTimeMS, e.RawHex) + sum := sha256.Sum256([]byte(input)) + return hex.EncodeToString(sum[:16]) +} + +func (e FrameEnvelope) KafkaKey() []byte { + return []byte(e.VehicleKey()) +} + +func (e FrameEnvelope) MarshalJSONBytes() ([]byte, error) { + if e.EventID == "" { + e.EventID = e.StableEventID() + } + if e.ParseStatus == "" { + e.ParseStatus = ParseOK + } + return json.Marshal(e) +} diff --git a/go/vehicle-gateway/internal/envelope/envelope_test.go b/go/vehicle-gateway/internal/envelope/envelope_test.go new file mode 100644 index 00000000..934d3bf8 --- /dev/null +++ b/go/vehicle-gateway/internal/envelope/envelope_test.go @@ -0,0 +1,55 @@ +package envelope + +import ( + "encoding/json" + "testing" +) + +func TestFrameEnvelopeVehicleKeyPrefersVIN(t *testing.T) { + e := FrameEnvelope{Protocol: ProtocolJT808, VIN: "LNBVIN00000000001", Phone: "013307795425"} + if got := e.VehicleKey(); got != "LNBVIN00000000001" { + t.Fatalf("VehicleKey() = %q", got) + } +} + +func TestFrameEnvelopeVehicleKeyFallsBackToPhone(t *testing.T) { + e := FrameEnvelope{Protocol: ProtocolJT808, Phone: "013307795425"} + if got := e.VehicleKey(); got != "JT808:013307795425" { + t.Fatalf("VehicleKey() = %q", got) + } +} + +func TestFrameEnvelopeEventIDStable(t *testing.T) { + e := FrameEnvelope{ + Protocol: ProtocolJT808, + MessageID: "0x0200", + Phone: "013307795425", + Sequence: 1, + EventTimeMS: 1782745114000, + ReceivedAtMS: 1782745114999, + RawHex: "7e02000000ff7e", + } + a := e.StableEventID() + b := e.StableEventID() + if a == "" || a != b { + t.Fatalf("event id must be non-empty and stable: %q %q", a, b) + } +} + +func TestFrameEnvelopeMarshalDefaults(t *testing.T) { + e := FrameEnvelope{Protocol: ProtocolGB32960, VIN: "LNBVIN00000000002", MessageID: "0x02"} + payload, err := e.MarshalJSONBytes() + if err != nil { + t.Fatalf("MarshalJSONBytes() error = %v", err) + } + var decoded FrameEnvelope + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if decoded.EventID == "" { + t.Fatal("event id should be filled") + } + if decoded.ParseStatus != ParseOK { + t.Fatalf("parse status = %q", decoded.ParseStatus) + } +} diff --git a/go/vehicle-gateway/internal/eventbus/kafka_sink.go b/go/vehicle-gateway/internal/eventbus/kafka_sink.go new file mode 100644 index 00000000..19dfbc6a --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/kafka_sink.go @@ -0,0 +1,94 @@ +package eventbus + +import ( + "context" + "errors" + "fmt" + + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +type Sink interface { + PublishRaw(context.Context, envelope.FrameEnvelope) error + PublishUnified(context.Context, envelope.FrameEnvelope) error + Close() error +} + +type KafkaSink struct { + writer kafkaWriter + rawTopics map[envelope.Protocol]string + unifiedTopic string +} + +type kafkaWriter interface { + WriteMessages(context.Context, ...kafka.Message) error + Close() error +} + +type KafkaConfig struct { + Brokers []string + RawTopics map[envelope.Protocol]string + UnifiedTopic string +} + +func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) { + if len(cfg.Brokers) == 0 { + return nil, errors.New("kafka brokers are required") + } + return newKafkaSinkWithWriter(&kafka.Writer{ + Addr: kafka.TCP(cfg.Brokers...), + Balancer: &kafka.Hash{}, + AllowAutoTopicCreation: false, + }, cfg), nil +} + +func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink { + rawTopics := map[envelope.Protocol]string{ + envelope.ProtocolGB32960: "vehicle.raw.gb32960.v1", + envelope.ProtocolJT808: "vehicle.raw.jt808.v1", + envelope.ProtocolYutongMQTT: "vehicle.raw.yutong-mqtt.v1", + } + for protocol, topic := range cfg.RawTopics { + if topic != "" { + rawTopics[protocol] = topic + } + } + unifiedTopic := cfg.UnifiedTopic + if unifiedTopic == "" { + unifiedTopic = "vehicle.event.unified.v1" + } + return &KafkaSink{writer: writer, rawTopics: rawTopics, unifiedTopic: unifiedTopic} +} + +func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { + topic, ok := s.rawTopics[env.Protocol] + if !ok || topic == "" { + return fmt.Errorf("raw topic not configured for protocol %s", env.Protocol) + } + return s.publish(ctx, topic, env) +} + +func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error { + return s.publish(ctx, s.unifiedTopic, env) +} + +func (s *KafkaSink) Close() error { + if s == nil || s.writer == nil { + return nil + } + return s.writer.Close() +} + +func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error { + payload, err := env.MarshalJSONBytes() + if err != nil { + return err + } + return s.writer.WriteMessages(ctx, kafka.Message{ + Topic: topic, + Key: env.KafkaKey(), + Value: payload, + }) +} diff --git a/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go b/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go new file mode 100644 index 00000000..a27fce47 --- /dev/null +++ b/go/vehicle-gateway/internal/eventbus/kafka_sink_test.go @@ -0,0 +1,77 @@ +package eventbus + +import ( + "context" + "encoding/json" + "testing" + + "github.com/segmentio/kafka-go" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestKafkaSinkRoutesRawTopics(t *testing.T) { + writer := &recordingWriter{} + sink := newKafkaSinkWithWriter(writer, KafkaConfig{}) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "013307795425", MessageID: "0x0200"} + if err := sink.PublishRaw(context.Background(), env); err != nil { + t.Fatalf("PublishRaw() error = %v", err) + } + if len(writer.messages) != 1 { + t.Fatalf("messages = %d", len(writer.messages)) + } + msg := writer.messages[0] + if msg.Topic != "vehicle.raw.jt808.v1" { + t.Fatalf("topic = %q", msg.Topic) + } + if string(msg.Key) != "JT808:013307795425" { + t.Fatalf("key = %q", string(msg.Key)) + } + var decoded envelope.FrameEnvelope + if err := json.Unmarshal(msg.Value, &decoded); err != nil { + t.Fatalf("json.Unmarshal() error = %v", err) + } + if decoded.EventID == "" || decoded.ParseStatus != envelope.ParseOK { + t.Fatalf("unexpected payload defaults: %#v", decoded) + } +} + +func TestKafkaSinkRoutesUnifiedTopic(t *testing.T) { + writer := &recordingWriter{} + sink := newKafkaSinkWithWriter(writer, KafkaConfig{UnifiedTopic: "custom.unified"}) + + env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBVIN00000000001", MessageID: "0x02"} + if err := sink.PublishUnified(context.Background(), env); err != nil { + t.Fatalf("PublishUnified() error = %v", err) + } + if writer.messages[0].Topic != "custom.unified" { + t.Fatalf("topic = %q", writer.messages[0].Topic) + } + if string(writer.messages[0].Key) != "LNBVIN00000000001" { + t.Fatalf("key = %q", string(writer.messages[0].Key)) + } +} + +func TestKafkaSinkRejectsUnknownProtocol(t *testing.T) { + writer := &recordingWriter{} + sink := newKafkaSinkWithWriter(writer, KafkaConfig{}) + + err := sink.PublishRaw(context.Background(), envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")}) + if err == nil { + t.Fatal("expected unknown protocol error") + } +} + +type recordingWriter struct { + messages []kafka.Message +} + +func (w *recordingWriter) WriteMessages(_ context.Context, messages ...kafka.Message) error { + w.messages = append(w.messages, messages...) + return nil +} + +func (w *recordingWriter) Close() error { + return nil +} diff --git a/go/vehicle-gateway/internal/observability/logger.go b/go/vehicle-gateway/internal/observability/logger.go new file mode 100644 index 00000000..ac54a1f8 --- /dev/null +++ b/go/vehicle-gateway/internal/observability/logger.go @@ -0,0 +1,13 @@ +package observability + +import ( + "log/slog" + "os" +) + +// NewLogger returns a JSON logger with a stable service field so ECS logs from +// different Go processes can be filtered without relying on container names. +func NewLogger(service string) *slog.Logger { + handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{AddSource: true}) + return slog.New(handler).With("service", service) +} diff --git a/go/vehicle-gateway/internal/protocol/gb32960/parser.go b/go/vehicle-gateway/internal/protocol/gb32960/parser.go new file mode 100644 index 00000000..8cbd6a1f --- /dev/null +++ b/go/vehicle-gateway/internal/protocol/gb32960/parser.go @@ -0,0 +1,237 @@ +package gb32960 + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +var ( + ErrFrameTooShort = errors.New("gb32960 frame too short") + ErrBadStartSymbol = errors.New("gb32960 bad start symbol") + ErrBodyLength = errors.New("gb32960 body length mismatch") + ErrBCC = errors.New("gb32960 bcc mismatch") +) + +const ( + minFrameLen = 25 + headerLen = 24 +) + +// ExtractFrames splits GB/T 32960 TCP streams using the protocol length field. +// Returned frames include the start symbols and trailing BCC byte. +func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) { + offset := 0 + for { + start := indexStart(stream[offset:]) + if start < 0 { + return frames, nil, nil + } + offset += start + remaining := stream[offset:] + if len(remaining) < headerLen { + return frames, append([]byte(nil), remaining...), nil + } + bodyLen := int(binary.BigEndian.Uint16(remaining[22:24])) + frameLen := headerLen + bodyLen + 1 + if len(remaining) < frameLen { + return frames, append([]byte(nil), remaining...), nil + } + frames = append(frames, append([]byte(nil), remaining[:frameLen]...)) + offset += frameLen + if offset >= len(stream) { + return frames, nil, nil + } + } +} + +func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) { + if len(raw) < minFrameLen { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw)) + } + version, ok := version(raw[0], raw[1]) + if !ok { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: %s", ErrBadStartSymbol, hex.EncodeToString(raw[:2])) + } + bodyLen := int(binary.BigEndian.Uint16(raw[22:24])) + if len(raw) != headerLen+bodyLen+1 { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: declared=%d actual=%d", ErrBodyLength, bodyLen, len(raw)-headerLen-1) + } + if got, want := bcc(raw[2:len(raw)-1]), raw[len(raw)-1]; got != want { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrBCC, got, want) + } + + command := raw[2] + responseFlag := raw[3] + vin := strings.TrimRight(string(raw[4:21]), "\x00 ") + body := raw[headerLen : headerLen+bodyLen] + parsed := map[string]any{ + "header": map[string]any{ + "version": version, + "command": fmt.Sprintf("0x%02X", command), + "response_flag": fmt.Sprintf("0x%02X", responseFlag), + "vin": vin, + "encrypt": raw[21], + "body_length": bodyLen, + }, + } + fields := map[string]any{} + eventTimeMS := receivedAtMS + + if command == 0x02 || command == 0x03 { + eventTime, units := parseDataBody(version, body, fields) + if !eventTime.IsZero() { + eventTimeMS = eventTime.UnixMilli() + fields["device_time"] = eventTime.Format(time.RFC3339) + parsed["device_time"] = eventTime.Format(time.RFC3339) + } + parsed["data_units"] = units + } + + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolGB32960, + MessageID: fmt.Sprintf("0x%02X", command), + VIN: vin, + SourceEndpoint: sourceEndpoint, + EventTimeMS: eventTimeMS, + ReceivedAtMS: receivedAtMS, + RawHex: strings.ToUpper(hex.EncodeToString(raw)), + Parsed: parsed, + Fields: fields, + ParseStatus: envelope.ParseOK, + } + env.EventID = env.StableEventID() + return env, nil +} + +func parseDataBody(version string, body []byte, fields map[string]any) (time.Time, []map[string]any) { + if len(body) < 6 { + return time.Time{}, nil + } + eventTime := parseBCDTime(body[:6]) + cursor := 6 + var units []map[string]any + for cursor < len(body) { + unitType := body[cursor] + cursor++ + switch unitType { + case 0x01: + size := 20 + if version == "V2025" { + size = 18 + } + if len(body[cursor:]) < size { + units = append(units, map[string]any{"type": "0x01", "error": "truncated"}) + return eventTime, units + } + unit := parseVehicleData(body[cursor : cursor+size]) + units = append(units, map[string]any{"type": "0x01", "name": "vehicle", "value": unit}) + fields["vehicle_status"] = unit["vehicle_status"] + fields["charge_status"] = unit["charge_status"] + fields["running_mode"] = unit["running_mode"] + fields[envelope.FieldSpeedKMH] = unit["speed_kmh"] + fields[envelope.FieldTotalMileageKM] = unit["total_mileage_km"] + fields[envelope.FieldSOCPercent] = unit["soc_percent"] + cursor += size + case 0x05: + if len(body[cursor:]) < 9 { + units = append(units, map[string]any{"type": "0x05", "error": "truncated"}) + return eventTime, units + } + unit := parsePositionData(body[cursor : cursor+9]) + units = append(units, map[string]any{"type": "0x05", "name": "position", "value": unit}) + fields["position_status"] = unit["position_status"] + fields[envelope.FieldLongitude] = unit["longitude"] + fields[envelope.FieldLatitude] = unit["latitude"] + cursor += 9 + default: + units = append(units, map[string]any{ + "type": fmt.Sprintf("0x%02X", unitType), + "raw_tail": strings.ToUpper(hex.EncodeToString(body[cursor:])), + "parse": "unsupported_unit", + "byte_size": len(body) - cursor, + }) + return eventTime, units + } + } + return eventTime, units +} + +func parseVehicleData(data []byte) map[string]any { + return map[string]any{ + "vehicle_status": int(data[0]), + "charge_status": int(data[1]), + "running_mode": int(data[2]), + "speed_kmh": float64(binary.BigEndian.Uint16(data[3:5])) / 10, + "total_mileage_km": float64(binary.BigEndian.Uint32(data[5:9])) / 10, + "total_voltage_v": float64(binary.BigEndian.Uint16(data[9:11])) / 10, + "total_current_a": float64(binary.BigEndian.Uint16(data[11:13]))/10 - 1000, + "soc_percent": int(data[13]), + "dc_dc_status": int(data[14]), + "gear": int(data[15]), + "insulation_kohm": binary.BigEndian.Uint16(data[16:18]), + "accelerator_pct": int(data[18]), + "brake_pct": int(data[19]), + } +} + +func parsePositionData(data []byte) map[string]any { + return map[string]any{ + "position_status": int(data[0]), + "longitude": float64(binary.BigEndian.Uint32(data[1:5])) / 1_000_000, + "latitude": float64(binary.BigEndian.Uint32(data[5:9])) / 1_000_000, + } +} + +func indexStart(data []byte) int { + for i := 0; i+1 < len(data); i++ { + if _, ok := version(data[i], data[i+1]); ok { + return i + } + } + return -1 +} + +func version(a byte, b byte) (string, bool) { + switch { + case a == '#' && b == '#': + return "V2016", true + case a == '$' && b == '$': + return "V2025", true + default: + return "", false + } +} + +func parseBCDTime(data []byte) time.Time { + if len(data) != 6 { + return time.Time{} + } + year := 2000 + bcdByte(data[0]) + month := time.Month(bcdByte(data[1])) + day := bcdByte(data[2]) + hour := bcdByte(data[3]) + minute := bcdByte(data[4]) + second := bcdByte(data[5]) + if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 { + return time.Time{} + } + return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600)) +} + +func bcdByte(value byte) int { + return int(value>>4)*10 + int(value&0x0f) +} + +func bcc(data []byte) byte { + var out byte + for _, value := range data { + out ^= value + } + return out +} diff --git a/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go b/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go new file mode 100644 index 00000000..75c80201 --- /dev/null +++ b/go/vehicle-gateway/internal/protocol/gb32960/parser_test.go @@ -0,0 +1,106 @@ +package gb32960 + +import ( + "encoding/hex" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +func TestExtractFramesKeepsPartialRemainderAndParsesHeader(t *testing.T) { + first := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57}) + second := buildFrame(0x05, 0xfe, "PLATFORM-LOGIN001", nil) + stream := append([]byte{0x00, 0x01}, first...) + stream = append(stream, second[:len(second)-3]...) + + frames, remainder, err := ExtractFrames(stream) + if err != nil { + t.Fatalf("ExtractFrames() error = %v", err) + } + if len(frames) != 1 { + t.Fatalf("expected one complete frame, got %d", len(frames)) + } + if string(remainder) != string(second[:len(second)-3]) { + t.Fatalf("expected partial second frame as remainder, got %s", hex.EncodeToString(remainder)) + } + + env, err := ParseFrame(frames[0], 1782745114999, "115.29.187.205:32960") + if err != nil { + t.Fatalf("ParseFrame() error = %v", err) + } + if env.Protocol != envelope.ProtocolGB32960 || env.MessageID != "0x02" { + t.Fatalf("unexpected envelope header: %#v", env) + } + if env.VIN != "LNBSCB3D4R1234567" { + t.Fatalf("unexpected vin: %q", env.VIN) + } +} + +func TestParseFrameRejectsBadBCC(t *testing.T) { + frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil) + frame[len(frame)-1] ^= 0xff + + if _, err := ParseFrame(frame, 1782745114999, ""); err == nil { + t.Fatal("expected bad bcc error") + } +} + +func TestParseFrameExtractsRealtimeVehicleMileageAndPosition(t *testing.T) { + body := []byte{0x1a, 0x06, 0x30, 0x16, 0x23, 0x57} + body = append(body, 0x01) + body = append(body, + 0x01, // vehicle status + 0x03, // charge status + 0x02, // running mode + 0x01, 0x2c, // speed: 30.0km/h + 0x00, 0x01, 0x86, 0xa0, // total mileage: 10000.0km + 0x15, 0xe5, // total voltage: 560.5V + 0x27, 0x10, // total current: 0A after -1000 offset + 85, 0x01, 0x00, + 0x27, 0x10, + 0x00, 0x00) + body = append(body, 0x05) + body = append(body, + 0x00, + 0x07, 0x36, 0x50, 0x40, // longitude: 121.000000 + 0x01, 0xd2, 0x4f, 0x00) // latitude: 30.560000 + frame := buildFrame(0x02, 0xfe, "LNBSCB3D4R1234567", body) + + env, err := ParseFrame(frame, 1782745114999, "115.29.187.205:32960") + if err != nil { + t.Fatalf("ParseFrame() error = %v", err) + } + assertFloatField(t, env, envelope.FieldSpeedKMH, 30.0) + assertFloatField(t, env, envelope.FieldTotalMileageKM, 10000.0) + assertFloatField(t, env, envelope.FieldLongitude, 121.0) + assertFloatField(t, env, envelope.FieldLatitude, 30.56) + if env.Fields[envelope.FieldSOCPercent] != 85 { + t.Fatalf("unexpected soc: %#v", env.Fields[envelope.FieldSOCPercent]) + } + if env.EventTimeMS == 1782745114999 { + t.Fatal("event time should come from GB32960 device timestamp") + } +} + +func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) { + t.Helper() + got, ok := env.Fields[key].(float64) + if !ok { + t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key]) + } + if got != want { + t.Fatalf("field %s = %v, want %v", key, got, want) + } +} + +func buildFrame(command byte, response byte, vin string, body []byte) []byte { + frame := []byte{'#', '#', command, response} + vinBytes := []byte(vin) + if len(vinBytes) < 17 { + vinBytes = append(vinBytes, make([]byte, 17-len(vinBytes))...) + } + frame = append(frame, vinBytes[:17]...) + frame = append(frame, 0x01, byte(len(body)>>8), byte(len(body))) + frame = append(frame, body...) + return append(frame, bcc(frame[2:])) +} diff --git a/go/vehicle-gateway/internal/protocol/jt808/parser.go b/go/vehicle-gateway/internal/protocol/jt808/parser.go new file mode 100644 index 00000000..e9a09729 --- /dev/null +++ b/go/vehicle-gateway/internal/protocol/jt808/parser.go @@ -0,0 +1,300 @@ +package jt808 + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +var ( + ErrFrameTooShort = errors.New("jt808 frame too short") + ErrChecksum = errors.New("jt808 checksum mismatch") + ErrEscape = errors.New("jt808 invalid escape sequence") +) + +type Location struct { + AlarmFlag uint32 + StatusFlag uint32 + Latitude float64 + Longitude float64 + AltitudeM uint16 + SpeedKMH float64 + DirectionDeg uint16 + Time time.Time + TotalMileageKM *float64 + Additional []AdditionalItem +} + +type AdditionalItem struct { + ID byte + Length int + ValueHex string + Parsed any +} + +// ExtractFrames splits JT/T 808 TCP streams by 0x7e delimiters and unescapes +// complete frame payloads. Returned frames exclude the leading and trailing 0x7e. +func ExtractFrames(stream []byte) (frames [][]byte, remainder []byte, err error) { + start := -1 + for i, value := range stream { + if value != 0x7e { + continue + } + if start < 0 { + start = i + continue + } + if i == start+1 { + start = i + continue + } + frame, err := unescape(stream[start+1 : i]) + if err != nil { + return nil, nil, err + } + frames = append(frames, frame) + start = i + } + if start >= 0 && start < len(stream)-1 { + return frames, append([]byte(nil), stream[start:]...), nil + } + return frames, nil, nil +} + +func ParseFrame(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) { + if len(raw) < 13 { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: len=%d", ErrFrameTooShort, len(raw)) + } + if got, want := checksum(raw[:len(raw)-1]), raw[len(raw)-1]; got != want { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: got=0x%02x want=0x%02x", ErrChecksum, got, want) + } + + messageID := binary.BigEndian.Uint16(raw[0:2]) + props := binary.BigEndian.Uint16(raw[2:4]) + bodySize := props & 0x03ff + headerLen := 12 + packageInfo := map[string]any(nil) + if props&0x2000 != 0 { + headerLen = 16 + packageInfo = map[string]any{ + "total": binary.BigEndian.Uint16(raw[12:14]), + "index": binary.BigEndian.Uint16(raw[14:16]), + } + } + if len(raw) < headerLen+int(bodySize)+1 { + return envelope.FrameEnvelope{}, fmt.Errorf("%w: bodySize=%d len=%d", ErrFrameTooShort, bodySize, len(raw)) + } + + phoneBCD := raw[4:10] + phone := bcdString(phoneBCD) + body := raw[headerLen : headerLen+int(bodySize)] + parsed := map[string]any{ + "header": map[string]any{ + "message_id": fmt.Sprintf("0x%04X", messageID), + "properties": props, + "body_size": bodySize, + "phone_bcd_hex": strings.ToUpper(hex.EncodeToString(phoneBCD)), + "phone": phone, + "phone_trim_zero": strings.TrimLeft(phone, "0"), + "sequence": binary.BigEndian.Uint16(raw[10:12]), + "subpackage": props&0x2000 != 0, + "package_info": packageInfo, + "encryption_flags": (props >> 10) & 0x07, + }, + } + fields := map[string]any{} + eventTimeMS := receivedAtMS + + if messageID == 0x0200 { + location, err := parseLocation(body) + if err != nil { + return envelope.FrameEnvelope{}, err + } + parsed["location"] = locationToMap(location) + fields[envelope.FieldLatitude] = location.Latitude + fields[envelope.FieldLongitude] = location.Longitude + fields[envelope.FieldSpeedKMH] = location.SpeedKMH + fields["altitude_m"] = location.AltitudeM + fields["direction_deg"] = location.DirectionDeg + fields["alarm_flag"] = location.AlarmFlag + fields["status_flag"] = location.StatusFlag + if !location.Time.IsZero() { + eventTimeMS = location.Time.UnixMilli() + fields["device_time"] = location.Time.Format(time.RFC3339) + } + if location.TotalMileageKM != nil { + fields[envelope.FieldTotalMileageKM] = *location.TotalMileageKM + } + } + + env := envelope.FrameEnvelope{ + Protocol: envelope.ProtocolJT808, + MessageID: fmt.Sprintf("0x%04X", messageID), + Sequence: binary.BigEndian.Uint16(raw[10:12]), + Phone: phone, + SourceEndpoint: sourceEndpoint, + EventTimeMS: eventTimeMS, + ReceivedAtMS: receivedAtMS, + RawHex: strings.ToUpper(hex.EncodeToString(raw)), + Parsed: parsed, + Fields: fields, + ParseStatus: envelope.ParseOK, + } + env.EventID = env.StableEventID() + return env, nil +} + +func parseLocation(body []byte) (Location, error) { + if len(body) < 28 { + return Location{}, fmt.Errorf("%w: location body len=%d", ErrFrameTooShort, len(body)) + } + location := Location{ + AlarmFlag: binary.BigEndian.Uint32(body[0:4]), + StatusFlag: binary.BigEndian.Uint32(body[4:8]), + Latitude: float64(binary.BigEndian.Uint32(body[8:12])) / 1_000_000, + Longitude: float64(binary.BigEndian.Uint32(body[12:16])) / 1_000_000, + AltitudeM: binary.BigEndian.Uint16(body[16:18]), + SpeedKMH: float64(binary.BigEndian.Uint16(body[18:20])) / 10, + DirectionDeg: binary.BigEndian.Uint16(body[20:22]), + Time: parseBCDTime(body[22:28]), + } + location.Additional = parseAdditional(body[28:], &location) + return location, nil +} + +func parseAdditional(data []byte, location *Location) []AdditionalItem { + var out []AdditionalItem + for len(data) >= 2 { + id := data[0] + size := int(data[1]) + data = data[2:] + if len(data) < size { + out = append(out, AdditionalItem{ + ID: id, + Length: size, + ValueHex: strings.ToUpper(hex.EncodeToString(data)), + Parsed: "truncated", + }) + return out + } + value := data[:size] + item := AdditionalItem{ + ID: id, + Length: size, + ValueHex: strings.ToUpper(hex.EncodeToString(value)), + } + // JT/T 808-2011 table 27: id 0x01 is GPS total mileage, DWORD, 0.1 km. + if id == 0x01 && size == 4 { + mileage := float64(binary.BigEndian.Uint32(value)) / 10 + location.TotalMileageKM = &mileage + item.Parsed = map[string]any{ + "name": envelope.FieldTotalMileageKM, + "value": mileage, + "unit": "km", + } + } + out = append(out, item) + data = data[size:] + } + return out +} + +func locationToMap(location Location) map[string]any { + out := map[string]any{ + "alarm_flag": location.AlarmFlag, + "status_flag": location.StatusFlag, + "latitude": location.Latitude, + "longitude": location.Longitude, + "altitude_m": location.AltitudeM, + "speed_kmh": location.SpeedKMH, + "direction_deg": location.DirectionDeg, + "additional": additionalToMaps(location.Additional), + } + if !location.Time.IsZero() { + out["device_time"] = location.Time.Format(time.RFC3339) + } + if location.TotalMileageKM != nil { + out[envelope.FieldTotalMileageKM] = *location.TotalMileageKM + } + return out +} + +func additionalToMaps(items []AdditionalItem) []map[string]any { + out := make([]map[string]any, 0, len(items)) + for _, item := range items { + value := map[string]any{ + "id": fmt.Sprintf("0x%02X", item.ID), + "length": item.Length, + "value_hex": item.ValueHex, + } + if item.Parsed != nil { + value["parsed"] = item.Parsed + } + out = append(out, value) + } + return out +} + +func bcdString(data []byte) string { + out := make([]byte, 0, len(data)*2) + for _, value := range data { + out = append(out, '0'+((value>>4)&0x0f), '0'+(value&0x0f)) + } + return string(out) +} + +func parseBCDTime(data []byte) time.Time { + if len(data) != 6 { + return time.Time{} + } + year := 2000 + bcdByte(data[0]) + month := time.Month(bcdByte(data[1])) + day := bcdByte(data[2]) + hour := bcdByte(data[3]) + minute := bcdByte(data[4]) + second := bcdByte(data[5]) + if month < 1 || month > 12 || day < 1 || day > 31 || hour > 23 || minute > 59 || second > 59 { + return time.Time{} + } + return time.Date(year, month, day, hour, minute, second, 0, time.FixedZone("Asia/Shanghai", 8*3600)) +} + +func bcdByte(value byte) int { + return int(value>>4)*10 + int(value&0x0f) +} + +func unescape(data []byte) ([]byte, error) { + out := make([]byte, 0, len(data)) + for i := 0; i < len(data); i++ { + if data[i] != 0x7d { + out = append(out, data[i]) + continue + } + if i+1 >= len(data) { + return nil, ErrEscape + } + i++ + switch data[i] { + case 0x01: + out = append(out, 0x7d) + case 0x02: + out = append(out, 0x7e) + default: + return nil, ErrEscape + } + } + return out, nil +} + +func checksum(data []byte) byte { + var out byte + for _, value := range data { + out ^= value + } + return out +} diff --git a/go/vehicle-gateway/internal/protocol/jt808/parser_test.go b/go/vehicle-gateway/internal/protocol/jt808/parser_test.go new file mode 100644 index 00000000..e1b587d8 --- /dev/null +++ b/go/vehicle-gateway/internal/protocol/jt808/parser_test.go @@ -0,0 +1,107 @@ +package jt808 + +import ( + "encoding/hex" + "testing" + + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" +) + +const sampleLocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E" + +func TestExtractFramesUnescapesAndParsesLocationWithTotalMileage(t *testing.T) { + stream, err := hex.DecodeString(sampleLocationFrame + "7E02000000") + if err != nil { + t.Fatal(err) + } + + frames, remainder, err := ExtractFrames(stream) + if err != nil { + t.Fatalf("ExtractFrames() error = %v", err) + } + if len(frames) != 1 { + t.Fatalf("expected one complete frame, got %d", len(frames)) + } + if hex.EncodeToString(remainder) != "7e02000000" { + t.Fatalf("expected partial remainder, got %s", hex.EncodeToString(remainder)) + } + + env, err := ParseFrame(frames[0], 1782745114999, "115.231.168.135:43625") + if err != nil { + t.Fatalf("ParseFrame() error = %v", err) + } + if env.Protocol != envelope.ProtocolJT808 { + t.Fatalf("protocol = %q", env.Protocol) + } + if env.MessageID != "0x0200" { + t.Fatalf("message id = %q", env.MessageID) + } + if env.Phone != "013307795425" { + t.Fatalf("phone = %q", env.Phone) + } + if env.Sequence != 1 { + t.Fatalf("sequence = %d", env.Sequence) + } + assertFloatField(t, env, envelope.FieldLatitude, 30.590151) + assertFloatField(t, env, envelope.FieldLongitude, 121.069881) + assertFloatField(t, env, envelope.FieldSpeedKMH, 23.0) + assertFloatField(t, env, envelope.FieldTotalMileageKM, 10241.2) + if env.Fields["direction_deg"] != uint16(79) { + t.Fatalf("direction = %#v", env.Fields["direction_deg"]) + } + + location, ok := env.Parsed["location"].(map[string]any) + if !ok { + t.Fatalf("parsed location missing: %#v", env.Parsed) + } + additional, ok := location["additional"].([]map[string]any) + if !ok || len(additional) == 0 { + t.Fatalf("additional fields missing: %#v", location["additional"]) + } + if additional[0]["id"] != "0x01" || additional[0]["value_hex"] != "0001900C" { + t.Fatalf("unexpected first additional item: %#v", additional[0]) + } +} + +func TestExtractFramesHandlesEscapedPayload(t *testing.T) { + payload := []byte{0x02, 0x00, 0x00, 0x02, 0x01, 0x33, 0x07, 0x79, 0x54, 0x25, 0x00, 0x01, 0x7e, 0x7d} + frame := append([]byte{0x7e}, escape(append(payload, checksum(payload)))...) + frame = append(frame, 0x7e) + + frames, remainder, err := ExtractFrames(frame) + if err != nil { + t.Fatalf("ExtractFrames() error = %v", err) + } + if len(frames) != 1 || len(remainder) != 0 { + t.Fatalf("unexpected split result frames=%d remainder=%s", len(frames), hex.EncodeToString(remainder)) + } + if hex.EncodeToString(frames[0][:len(payload)]) != hex.EncodeToString(payload) { + t.Fatalf("frame was not unescaped: %s", hex.EncodeToString(frames[0])) + } +} + +func assertFloatField(t *testing.T, env envelope.FrameEnvelope, key string, want float64) { + t.Helper() + got, ok := env.Fields[key].(float64) + if !ok { + t.Fatalf("field %s missing or not float64: %#v", key, env.Fields[key]) + } + if got != want { + t.Fatalf("field %s = %v, want %v", key, got, want) + } +} + +func escape(payload []byte) []byte { + var out []byte + for _, value := range payload { + switch value { + case 0x7e: + out = append(out, 0x7d, 0x02) + case 0x7d: + out = append(out, 0x7d, 0x01) + default: + out = append(out, value) + } + } + return out +}