shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
go/vehicle-gateway/internal/loadsim/config.go
View File

@@ -14,6 +14,9 @@ const (
ProtocolGB32960 Protocol = "gb32960"
ProtocolJT808 Protocol = "jt808"
ProtocolYutongMQTT Protocol = "yutong-mqtt"
DefaultJT808PhoneBase int64 = 139000000000
maxJT808Phone int64 = 999999999999
)
type Config struct {
@@ -25,17 +28,21 @@ type Config struct {
Duration time.Duration
Template string
SendFrames bool
JT808PhoneBase int64
DrainResponses bool
}
type FlagConfig struct {
protocol string
addr string
connections int
connectRate int
sendInterval time.Duration
duration time.Duration
template string
sendFrames bool
protocol string
addr string
connections int
connectRate int
sendInterval time.Duration
duration time.Duration
template string
sendFrames bool
jt808PhoneBase int64
drainResponses bool
}
func RegisterFlags(fs *flag.FlagSet) *FlagConfig {
@@ -48,6 +55,8 @@ func RegisterFlags(fs *flag.FlagSet) *FlagConfig {
fs.DurationVar(&cfg.duration, "duration", 10*time.Minute, "load test duration")
fs.StringVar(&cfg.template, "template", "", "frame template name")
fs.BoolVar(&cfg.sendFrames, "send", true, "send protocol frames while connections are open")
fs.Int64Var(&cfg.jt808PhoneBase, "jt808-phone-base", DefaultJT808PhoneBase, "first synthetic JT808 phone; one consecutive phone is assigned per connection")
fs.BoolVar(&cfg.drainResponses, "drain-responses", true, "continuously read protocol responses while frames are being sent")
return cfg
}
@@ -73,6 +82,14 @@ func (c *FlagConfig) Build() (Config, error) {
if c.duration <= 0 {
return Config{}, errors.New("duration must be positive")
}
if protocol == ProtocolJT808 {
if c.jt808PhoneBase <= 0 || c.jt808PhoneBase > maxJT808Phone {
return Config{}, errors.New("jt808-phone-base must be a positive 12-digit-or-shorter number")
}
if int64(c.connections-1) > maxJT808Phone-c.jt808PhoneBase {
return Config{}, errors.New("jt808 synthetic phone range exceeds 12 digits")
}
}
return Config{
Protocol: protocol,
Addr: strings.TrimSpace(c.addr),
@@ -82,5 +99,7 @@ func (c *FlagConfig) Build() (Config, error) {
Duration: c.duration,
Template: strings.TrimSpace(c.template),
SendFrames: c.sendFrames,
JT808PhoneBase: c.jt808PhoneBase,
DrainResponses: c.drainResponses,
}, nil
}

8
go/vehicle-gateway/internal/loadsim/config_test.go
View File

@@ -18,6 +18,7 @@ func TestConfigFromFlagsParsesCapacityKnobs(t *testing.T) {
"-send-interval", "5s",
"-duration", "30m",
"-template", "0200",
"-jt808-phone-base", "138900000000",
"-send=false",
})
if err != nil {
@@ -53,6 +54,12 @@ func TestConfigFromFlagsParsesCapacityKnobs(t *testing.T) {
if got.SendFrames {
t.Fatal("SendFrames = true, want false")
}
if got.JT808PhoneBase != 138900000000 {
t.Fatalf("JT808PhoneBase = %d, want 138900000000", got.JT808PhoneBase)
}
if !got.DrainResponses {
t.Fatal("DrainResponses = false, want true by default")
}
}
func TestConfigFromFlagsRejectsUnsafeValues(t *testing.T) {
@@ -62,6 +69,7 @@ func TestConfigFromFlagsRejectsUnsafeValues(t *testing.T) {
"zero connections": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "0"},
"zero connect rate": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-connect-rate", "0"},
"zero interval": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-send-interval", "0s"},
"phone overflow": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "2", "-jt808-phone-base", "999999999999"},
} {
t.Run(name, func(t *testing.T) {
fs := flag.NewFlagSet("load-sim", flag.ContinueOnError)

20
go/vehicle-gateway/internal/loadsim/mutator.go
View File

@@ -7,22 +7,30 @@ import (
)
type FrameFactory struct {
protocol Protocol
base []byte
protocol Protocol
base []byte
jt808PhoneBase int64
}
func NewFrameFactory(protocol Protocol, template string) (*FrameFactory, error) {
return NewFrameFactoryWithJT808PhoneBase(protocol, template, DefaultJT808PhoneBase)
}
func NewFrameFactoryWithJT808PhoneBase(protocol Protocol, template string, phoneBase int64) (*FrameFactory, error) {
base, err := FrameTemplate(protocol, template)
if err != nil {
return nil, err
}
return &FrameFactory{protocol: protocol, base: base}, nil
if phoneBase <= 0 {
phoneBase = DefaultJT808PhoneBase
}
return &FrameFactory{protocol: protocol, base: base, jt808PhoneBase: phoneBase}, nil
}
func (f *FrameFactory) Frame(connectionIndex int, frameIndex int64) ([]byte, error) {
switch f.protocol {
case ProtocolJT808:
return mutateJT808Frame(f.base, connectionIndex, frameIndex)
return mutateJT808Frame(f.base, f.jt808PhoneBase, connectionIndex, frameIndex)
case ProtocolGB32960:
return mutateGB32960Frame(f.base, connectionIndex, frameIndex)
default:
@@ -31,7 +39,7 @@ func (f *FrameFactory) Frame(connectionIndex int, frameIndex int64) ([]byte, err
}
}
func mutateJT808Frame(base []byte, connectionIndex int, frameIndex int64) ([]byte, error) {
func mutateJT808Frame(base []byte, phoneBase int64, connectionIndex int, frameIndex int64) ([]byte, error) {
if len(base) < 2 || base[0] != 0x7e || base[len(base)-1] != 0x7e {
return nil, fmt.Errorf("jt808 template must include 0x7e delimiters")
}
@@ -42,7 +50,7 @@ func mutateJT808Frame(base []byte, connectionIndex int, frameIndex int64) ([]byt
if len(payload) < 13 {
return nil, fmt.Errorf("jt808 template too short: %d", len(payload))
}
phone := fmt.Sprintf("%012d", 139000000000+connectionIndex%100000000)
phone := fmt.Sprintf("%012d", phoneBase+int64(connectionIndex))
copy(payload[4:10], encodeBCD(phone, 6))
binary.BigEndian.PutUint16(payload[10:12], uint16((int(frameIndex)+connectionIndex)%65536))
bodySize := int(binary.BigEndian.Uint16(payload[2:4]) & 0x03ff)

14
go/vehicle-gateway/internal/loadsim/mutator_test.go
View File

@@ -38,6 +38,20 @@ func TestFrameFactoryGeneratesUniqueParsableJT808Frames(t *testing.T) {
}
}
func TestFrameFactoryUsesConfiguredJT808PhoneBase(t *testing.T) {
factory, err := NewFrameFactoryWithJT808PhoneBase(ProtocolJT808, "0200", 138900000000)
if err != nil {
t.Fatalf("NewFrameFactoryWithJT808PhoneBase() error = %v", err)
}
frame, err := factory.Frame(42, 0)
if err != nil {
t.Fatalf("Frame() error = %v", err)
}
if got := parseJT808Frame(t, frame).Phone; got != "138900000042" {
t.Fatalf("phone = %q, want 138900000042", got)
}
}
func TestFrameFactoryGeneratesUniqueParsableGB32960Frames(t *testing.T) {
factory, err := NewFrameFactory(ProtocolGB32960, "realtime")
if err != nil {

62
go/vehicle-gateway/internal/loadsim/runner.go
View File

@@ -2,6 +2,8 @@ package loadsim
import (
"context"
"errors"
"io"
"net"
"sync"
"sync/atomic"
@@ -19,24 +21,32 @@ type Stats struct {
ConnectionsFailed int64
FramesWritten int64
WriteErrors int64
ResponseBytes int64
ReadErrors int64
}
func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) {
phoneBase := cfg.JT808PhoneBase
if phoneBase <= 0 {
phoneBase = DefaultJT808PhoneBase
}
if _, err := (&FlagConfig{
protocol: string(cfg.Protocol),
addr: cfg.Addr,
connections: cfg.Connections,
connectRate: cfg.ConnectRatePerSecond,
sendInterval: cfg.SendInterval,
duration: cfg.Duration,
template: cfg.Template,
protocol: string(cfg.Protocol),
addr: cfg.Addr,
connections: cfg.Connections,
connectRate: cfg.ConnectRatePerSecond,
sendInterval: cfg.SendInterval,
duration: cfg.Duration,
template: cfg.Template,
jt808PhoneBase: phoneBase,
drainResponses: cfg.DrainResponses,
}).Build(); err != nil {
return Stats{}, err
}
var factory *FrameFactory
if cfg.SendFrames {
var err error
factory, err = NewFrameFactory(cfg.Protocol, cfg.Template)
factory, err = NewFrameFactoryWithJT808PhoneBase(cfg.Protocol, cfg.Template, phoneBase)
if err != nil {
return Stats{}, err
}
@@ -74,12 +84,24 @@ func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) {
connectionIndex++
go func() {
defer wg.Done()
defer conn.Close()
if cfg.SendFrames {
var readDone chan struct{}
if cfg.DrainResponses {
readDone = make(chan struct{})
go func() {
drainResponses(runCtx, conn, &stats)
close(readDone)
}()
}
writeLoop(runCtx, conn, factory, connIndex, cfg.SendInterval, &stats)
_ = conn.Close()
if readDone != nil {
<-readDone
}
return
}
<-runCtx.Done()
_ = conn.Close()
}()
}
}
@@ -88,6 +110,28 @@ func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) {
return stats, nil
}
func drainResponses(ctx context.Context, conn net.Conn, stats *Stats) {
buffer := make([]byte, 4096)
for {
_ = conn.SetReadDeadline(time.Now().Add(time.Second))
read, err := conn.Read(buffer)
if read > 0 {
atomic.AddInt64(&stats.ResponseBytes, int64(read))
}
if err == nil {
continue
}
if ctx.Err() != nil || errors.Is(err, net.ErrClosed) || errors.Is(err, io.EOF) {
return
}
if timeout, ok := err.(net.Error); ok && timeout.Timeout() {
continue
}
atomic.AddInt64(&stats.ReadErrors, 1)
return
}
}
func writeLoop(ctx context.Context, conn net.Conn, factory *FrameFactory, connectionIndex int, interval time.Duration, stats *Stats) {
for frameIndex := int64(0); ; frameIndex++ {
payload, err := factory.Frame(connectionIndex, frameIndex)

42
go/vehicle-gateway/internal/loadsim/runner_test.go
View File

@@ -131,6 +131,48 @@ func TestRunnerCanHoldConnectionsWithoutWritingFrames(t *testing.T) {
}
}
func TestRunnerDrainsProtocolResponses(t *testing.T) {
runner := Runner{
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
client, server := net.Pipe()
go func() {
defer server.Close()
buffer := make([]byte, 4096)
for {
if _, err := server.Read(buffer); err != nil {
return
}
if _, err := server.Write([]byte{0x7e, 0x80, 0x01, 0x7e}); err != nil {
return
}
}
}()
return client, nil
},
}
stats, err := runner.Run(context.Background(), Config{
Protocol: ProtocolJT808,
Addr: "127.0.0.1:808",
Connections: 1,
ConnectRatePerSecond: 1000,
SendInterval: time.Millisecond,
Duration: 10 * time.Millisecond,
Template: "0200",
SendFrames: true,
DrainResponses: true,
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if stats.ResponseBytes == 0 {
t.Fatalf("ResponseBytes = 0, want drained protocol responses")
}
if stats.ReadErrors != 0 {
t.Fatalf("ReadErrors = %d, want 0", stats.ReadErrors)
}
}
type recordingConn struct {
write func([]byte) (int, error)
close func() error