shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): add 100k ingest hardening baseline

Browse Source
This commit is contained in:
lingniu
2026-07-03 17:55:22 +08:00
parent 378a5d5e57
commit 79e591f864
20 changed files with 1360 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go/vehicle-gateway/cmd/gateway/main.go
View File

@@ -74,7 +74,7 @@ func main() {
Metrics: registry,
ReadBufferSize: envInt("TCP_READ_BUFFER_BYTES", 64*1024),
IdleTimeout: time.Duration(envInt("TCP_IDLE_TIMEOUT_SECONDS", 180)) * time.Second,
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 20_000),
MaxConnections: envInt("TCP_MAX_CONNECTIONS", 120_000),
PublishUnified: publishUnified,
})
if err != nil {

10
go/vehicle-gateway/cmd/gateway/main_test.go
View File

@@ -46,6 +46,16 @@ func TestGatewayConfiguresIdentityLookupCacheTTL(t *testing.T) {
}
}
func TestGatewayDefaultsTo100KConnectionCeiling(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
t.Fatalf("read main.go: %v", err)
}
if !strings.Contains(string(source), `envInt("TCP_MAX_CONNECTIONS", 120_000)`) {
t.Fatal("gateway should default TCP_MAX_CONNECTIONS to a 100K-ready ceiling")
}
}
func TestNATSSinkConfigFromEnvDefaultsToGoSubjects(t *testing.T) {
t.Setenv("NATS_URL", "nats://172.17.111.56:4222")

45
go/vehicle-gateway/cmd/load-sim/main.go Normal file
View File

@@ -0,0 +1,45 @@
package main
import (
"context"
"flag"
"fmt"
"log"
"os"
"os/signal"
"syscall"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/loadsim"
)
func main() {
log.SetFlags(log.LstdFlags | log.Lmicroseconds)
flagCfg := loadsim.RegisterFlags(flag.CommandLine)
if err := flag.CommandLine.Parse(os.Args[1:]); err != nil {
log.Fatal(err)
}
cfg, err := flagCfg.Build()
if err != nil {
log.Fatal(err)
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
log.Printf("load simulation started protocol=%s addr=%s connections=%d connect_rate=%d send_interval=%s duration=%s template=%s",
cfg.Protocol, cfg.Addr, cfg.Connections, cfg.ConnectRatePerSecond, cfg.SendInterval, cfg.Duration, cfg.Template)
stats, err := (loadsim.Runner{}).Run(ctx, cfg)
if err != nil {
log.Fatal(err)
}
log.Print(formatStats(stats))
}
func formatStats(stats loadsim.Stats) string {
return fmt.Sprintf("connections_opened=%d connections_failed=%d frames_written=%d write_errors=%d",
stats.ConnectionsOpened,
stats.ConnectionsFailed,
stats.FramesWritten,
stats.WriteErrors,
)
}

28
go/vehicle-gateway/cmd/load-sim/main_test.go Normal file
View File

@@ -0,0 +1,28 @@
package main
import (
"strings"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/loadsim"
)
func TestFormatStatsIncludesCapacityCounters(t *testing.T) {
out := formatStats(loadsim.Stats{
ConnectionsOpened: 10,
ConnectionsFailed: 2,
FramesWritten: 300,
WriteErrors: 1,
})
for _, want := range []string{
"connections_opened=10",
"connections_failed=2",
"frames_written=300",
"write_errors=1",
} {
if !strings.Contains(out, want) {
t.Fatalf("formatStats() = %q, missing %q", out, want)
}
}
}

29
go/vehicle-gateway/cmd/realtime-api/main_test.go
View File

@@ -134,6 +134,35 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) {
}
}
func TestAsyncSecondaryQueueDropDoesNotFailPrimaryUpdate(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"}
primary := &contextCheckingRealtimeUpdater{}
registry := metrics.NewRegistry()
updater := &asyncSecondaryRealtimeUpdater{
primary: primary,
secondary: &contextCheckingRealtimeUpdater{},
queue: make(chan envelope.FrameEnvelope, 1),
registry: registry,
}
updater.queue <- envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN000"}
if err := updater.Update(context.Background(), env); err != nil {
t.Fatalf("Update() error = %v", err)
}
if primary.count != 1 {
t.Fatalf("primary updates = %d, want 1", primary.count)
}
text := registry.Render()
for _, want := range []string{
`vehicle_realtime_async_queue_total{protocol="JT808",status="dropped",store="mysql"} 1`,
`vehicle_realtime_async_queue_depth{protocol="JT808",store="mysql"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("async queue drop metric missing %s:\n%s", want, text)
}
}
}
func TestStoreMetricUpdaterRecordsStoreUpdateResults(t *testing.T) {
registry := metrics.NewRegistry()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"}

11
go/vehicle-gateway/internal/gateway/tcp_server.go
View File

@@ -143,6 +143,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
s.handleConnection(ctx, conn)
}()
default:
s.recordConnectionRejection("max_connections")
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
_ = conn.Close()
}
@@ -367,6 +368,16 @@ func (s *TCPServer) recordConnectionMetric(delta float64) {
}, delta)
}
func (s *TCPServer) recordConnectionRejection(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_connection_rejections_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func identityStatus(env envelope.FrameEnvelope) string {
if strings.TrimSpace(env.VIN) != "" {
return "resolved"

24
go/vehicle-gateway/internal/gateway/tcp_server_test.go
View File

@@ -137,6 +137,30 @@ func TestTCPServerRecordsActiveConnectionGauge(t *testing.T) {
}
}
func TestTCPServerRecordsConnectionRejectionMetric(t *testing.T) {
registry := metrics.NewRegistry()
server, err := NewTCPServer(TCPServerConfig{
Protocol: TCPProtocol{
Protocol: envelope.ProtocolJT808,
Addr: ":0",
Extract: jt808.ExtractFrames,
Parse: jt808.ParseFrame,
},
Sink: &recordingSink{},
Logger: slog.New(slog.NewTextHandler(testWriter{t: t}, nil)),
Metrics: registry,
})
if err != nil {
t.Fatalf("NewTCPServer() error = %v", err)
}
server.recordConnectionRejection("max_connections")
if text := registry.Render(); !strings.Contains(text, `vehicle_gateway_connection_rejections_total{protocol="JT808",reason="max_connections"} 1`) {
t.Fatalf("connection rejection metric missing:\n%s", text)
}
}
func TestTCPServerPublishesBadFrameOnlyToRaw(t *testing.T) {
good := buildGBFrame(0x02, 0xfe, "LNBSCB3D4R1234567", nil)
good[len(good)-1] ^= 0xff

82
go/vehicle-gateway/internal/loadsim/config.go Normal file
View File

@@ -0,0 +1,82 @@
package loadsim
import (
"errors"
"flag"
"fmt"
"strings"
"time"
)
type Protocol string
const (
ProtocolGB32960 Protocol = "gb32960"
ProtocolJT808 Protocol = "jt808"
ProtocolYutongMQTT Protocol = "yutong-mqtt"
)
type Config struct {
Protocol Protocol
Addr string
Connections int
ConnectRatePerSecond int
SendInterval time.Duration
Duration time.Duration
Template string
}
type FlagConfig struct {
protocol string
addr string
connections int
connectRate int
sendInterval time.Duration
duration time.Duration
template string
}
func RegisterFlags(fs *flag.FlagSet) *FlagConfig {
cfg := &FlagConfig{}
fs.StringVar(&cfg.protocol, "protocol", string(ProtocolJT808), "protocol: jt808, gb32960, yutong-mqtt")
fs.StringVar(&cfg.addr, "addr", "", "target host:port")
fs.IntVar(&cfg.connections, "connections", 1, "target concurrent connections")
fs.IntVar(&cfg.connectRate, "connect-rate", 100, "new connections per second")
fs.DurationVar(&cfg.sendInterval, "send-interval", 10*time.Second, "frame send interval per connection")
fs.DurationVar(&cfg.duration, "duration", 10*time.Minute, "load test duration")
fs.StringVar(&cfg.template, "template", "", "frame template name")
return cfg
}
func (c *FlagConfig) Build() (Config, error) {
protocol := Protocol(strings.ToLower(strings.TrimSpace(c.protocol)))
switch protocol {
case ProtocolGB32960, ProtocolJT808, ProtocolYutongMQTT:
default:
return Config{}, fmt.Errorf("unsupported protocol %q", c.protocol)
}
if strings.TrimSpace(c.addr) == "" {
return Config{}, errors.New("addr is required")
}
if c.connections <= 0 {
return Config{}, errors.New("connections must be positive")
}
if c.connectRate <= 0 {
return Config{}, errors.New("connect-rate must be positive")
}
if c.sendInterval <= 0 {
return Config{}, errors.New("send-interval must be positive")
}
if c.duration <= 0 {
return Config{}, errors.New("duration must be positive")
}
return Config{
Protocol: protocol,
Addr: strings.TrimSpace(c.addr),
Connections: c.connections,
ConnectRatePerSecond: c.connectRate,
SendInterval: c.sendInterval,
Duration: c.duration,
Template: strings.TrimSpace(c.template),
}, nil
}

73
go/vehicle-gateway/internal/loadsim/config_test.go Normal file
View File

@@ -0,0 +1,73 @@
package loadsim
import (
"flag"
"testing"
"time"
)
func TestConfigFromFlagsParsesCapacityKnobs(t *testing.T) {
fs := flag.NewFlagSet("load-sim", flag.ContinueOnError)
cfg := RegisterFlags(fs)
err := fs.Parse([]string{
"-protocol", "jt808",
"-addr", "115.29.187.205:808",
"-connections", "10000",
"-connect-rate", "800",
"-send-interval", "5s",
"-duration", "30m",
"-template", "0200",
})
if err != nil {
t.Fatalf("parse flags: %v", err)
}
got, err := cfg.Build()
if err != nil {
t.Fatalf("build config: %v", err)
}
if got.Protocol != ProtocolJT808 {
t.Fatalf("Protocol = %q, want %q", got.Protocol, ProtocolJT808)
}
if got.Addr != "115.29.187.205:808" {
t.Fatalf("Addr = %q", got.Addr)
}
if got.Connections != 10000 {
t.Fatalf("Connections = %d", got.Connections)
}
if got.ConnectRatePerSecond != 800 {
t.Fatalf("ConnectRatePerSecond = %d", got.ConnectRatePerSecond)
}
if got.SendInterval != 5*time.Second {
t.Fatalf("SendInterval = %s", got.SendInterval)
}
if got.Duration != 30*time.Minute {
t.Fatalf("Duration = %s", got.Duration)
}
if got.Template != "0200" {
t.Fatalf("Template = %q", got.Template)
}
}
func TestConfigFromFlagsRejectsUnsafeValues(t *testing.T) {
for name, args := range map[string][]string{
"missing addr": {"-protocol", "jt808", "-connections", "1"},
"bad protocol": {"-protocol", "x", "-addr", "127.0.0.1:808", "-connections", "1"},
"zero connections": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "0"},
"zero connect rate": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-connect-rate", "0"},
"zero interval": {"-protocol", "jt808", "-addr", "127.0.0.1:808", "-connections", "1", "-send-interval", "0s"},
} {
t.Run(name, func(t *testing.T) {
fs := flag.NewFlagSet("load-sim", flag.ContinueOnError)
cfg := RegisterFlags(fs)
if err := fs.Parse(args); err != nil {
t.Fatalf("parse flags: %v", err)
}
if _, err := cfg.Build(); err == nil {
t.Fatalf("Build() error = nil, want validation error")
}
})
}
}

103
go/vehicle-gateway/internal/loadsim/runner.go Normal file
View File

@@ -0,0 +1,103 @@
package loadsim
import (
"context"
"net"
"sync"
"sync/atomic"
"time"
)
type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
type Runner struct {
Dial DialFunc
}
type Stats struct {
ConnectionsOpened int64
ConnectionsFailed int64
FramesWritten int64
WriteErrors int64
}
func (r Runner) Run(ctx context.Context, cfg Config) (Stats, error) {
if _, err := (&FlagConfig{
protocol: string(cfg.Protocol),
addr: cfg.Addr,
connections: cfg.Connections,
connectRate: cfg.ConnectRatePerSecond,
sendInterval: cfg.SendInterval,
duration: cfg.Duration,
template: cfg.Template,
}).Build(); err != nil {
return Stats{}, err
}
payload, err := FrameTemplate(cfg.Protocol, cfg.Template)
if err != nil {
return Stats{}, err
}
dial := r.Dial
if dial == nil {
dialer := &net.Dialer{Timeout: 5 * time.Second, KeepAlive: 30 * time.Second}
dial = dialer.DialContext
}
runCtx, cancel := context.WithTimeout(ctx, cfg.Duration)
defer cancel()
var stats Stats
var wg sync.WaitGroup
for i, batch := range ConnectionBatches(cfg.Connections, cfg.ConnectRatePerSecond) {
if i > 0 {
if !sleepOrDone(runCtx, time.Second) {
break
}
}
for range batch {
if runCtx.Err() != nil {
break
}
conn, err := dial(runCtx, "tcp", cfg.Addr)
if err != nil {
atomic.AddInt64(&stats.ConnectionsFailed, 1)
continue
}
atomic.AddInt64(&stats.ConnectionsOpened, 1)
wg.Add(1)
go func() {
defer wg.Done()
defer conn.Close()
writeLoop(runCtx, conn, payload, cfg.SendInterval, &stats)
}()
}
}
<-runCtx.Done()
wg.Wait()
return stats, nil
}
func writeLoop(ctx context.Context, conn net.Conn, payload []byte, interval time.Duration, stats *Stats) {
for {
_ = conn.SetWriteDeadline(time.Now().Add(3 * time.Second))
if _, err := conn.Write(payload); err != nil {
atomic.AddInt64(&stats.WriteErrors, 1)
return
}
atomic.AddInt64(&stats.FramesWritten, 1)
if !sleepOrDone(ctx, interval) {
return
}
}
}
func sleepOrDone(ctx context.Context, duration time.Duration) bool {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return false
case <-timer.C:
return true
}
}

73
go/vehicle-gateway/internal/loadsim/runner_test.go Normal file
View File

@@ -0,0 +1,73 @@
package loadsim
import (
"context"
"net"
"sync/atomic"
"testing"
"time"
)
func TestRunnerDialsTargetConnectionsAndWritesFrames(t *testing.T) {
var writes atomic.Int64
var closed atomic.Int64
runner := Runner{
Dial: func(ctx context.Context, network, addr string) (net.Conn, error) {
return &recordingConn{
write: func(p []byte) (int, error) {
writes.Add(1)
return len(p), nil
},
close: func() error {
closed.Add(1)
return nil
},
}, nil
},
}
stats, err := runner.Run(context.Background(), Config{
Protocol: ProtocolJT808,
Addr: "127.0.0.1:808",
Connections: 3,
ConnectRatePerSecond: 1000,
SendInterval: time.Millisecond,
Duration: 5 * time.Millisecond,
Template: "0200",
})
if err != nil {
t.Fatalf("Run() error = %v", err)
}
if stats.ConnectionsOpened != 3 {
t.Fatalf("ConnectionsOpened = %d, want 3", stats.ConnectionsOpened)
}
if stats.FramesWritten == 0 {
t.Fatalf("FramesWritten = 0, want at least one")
}
if writes.Load() != int64(stats.FramesWritten) {
t.Fatalf("recorded writes = %d, stats = %d", writes.Load(), stats.FramesWritten)
}
if closed.Load() != 3 {
t.Fatalf("closed = %d, want 3", closed.Load())
}
}
type recordingConn struct {
write func([]byte) (int, error)
close func() error
}
func (c *recordingConn) Read(p []byte) (int, error) { return 0, context.Canceled }
func (c *recordingConn) Write(p []byte) (int, error) { return c.write(p) }
func (c *recordingConn) Close() error { return c.close() }
func (c *recordingConn) LocalAddr() net.Addr { return testAddr("local") }
func (c *recordingConn) RemoteAddr() net.Addr { return testAddr("remote") }
func (c *recordingConn) SetDeadline(t time.Time) error { return nil }
func (c *recordingConn) SetReadDeadline(t time.Time) error { return nil }
func (c *recordingConn) SetWriteDeadline(t time.Time) error { return nil }
type testAddr string
func (a testAddr) Network() string { return "tcp" }
func (a testAddr) String() string { return string(a) }

18
go/vehicle-gateway/internal/loadsim/schedule.go Normal file
View File

@@ -0,0 +1,18 @@
package loadsim
func ConnectionBatches(total int, ratePerSecond int) []int {
if total <= 0 || ratePerSecond <= 0 {
return nil
}
batches := make([]int, 0, (total+ratePerSecond-1)/ratePerSecond)
remaining := total
for remaining > 0 {
batch := ratePerSecond
if remaining < batch {
batch = remaining
}
batches = append(batches, batch)
remaining -= batch
}
return batches
}

24
go/vehicle-gateway/internal/loadsim/schedule_test.go Normal file
View File

@@ -0,0 +1,24 @@
package loadsim
import "testing"
func TestConnectionBatchesSpreadsConnectionsByRate(t *testing.T) {
got := ConnectionBatches(2500, 1000)
want := []int{1000, 1000, 500}
if len(got) != len(want) {
t.Fatalf("len = %d, want %d: %#v", len(got), len(want), got)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("batch[%d] = %d, want %d: %#v", i, got[i], want[i], got)
}
}
}
func TestConnectionBatchesUsesOneBatchWhenTargetBelowRate(t *testing.T) {
got := ConnectionBatches(300, 1000)
if len(got) != 1 || got[0] != 300 {
t.Fatalf("ConnectionBatches() = %#v, want []int{300}", got)
}
}

50
go/vehicle-gateway/internal/loadsim/template_test.go Normal file
View File

@@ -0,0 +1,50 @@
package loadsim
import (
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/gb32960"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/protocol/jt808"
)
func TestFrameTemplateReturnsParsableJT808LocationFrame(t *testing.T) {
frame, err := FrameTemplate(ProtocolJT808, "0200")
if err != nil {
t.Fatalf("FrameTemplate() error = %v", err)
}
frames, remainder, err := jt808.ExtractFrames(frame)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(remainder) != 0 {
t.Fatalf("remainder len = %d", len(remainder))
}
if len(frames) != 1 {
t.Fatalf("frames len = %d", len(frames))
}
if _, err := jt808.ParseFrame(frames[0], 1782918600000, "127.0.0.1:808"); err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
}
func TestFrameTemplateReturnsParsableGB32960RealtimeFrame(t *testing.T) {
frame, err := FrameTemplate(ProtocolGB32960, "realtime")
if err != nil {
t.Fatalf("FrameTemplate() error = %v", err)
}
frames, remainder, err := gb32960.ExtractFrames(frame)
if err != nil {
t.Fatalf("ExtractFrames() error = %v", err)
}
if len(remainder) != 0 {
t.Fatalf("remainder len = %d", len(remainder))
}
if len(frames) != 1 {
t.Fatalf("frames len = %d", len(frames))
}
if _, err := gb32960.ParseFrame(frames[0], 1782918600000, "127.0.0.1:32960"); err != nil {
t.Fatalf("ParseFrame() error = %v", err)
}
}

53
go/vehicle-gateway/internal/loadsim/templates.go Normal file
View File

@@ -0,0 +1,53 @@
package loadsim
import (
"encoding/hex"
"fmt"
"strings"
)
const sampleJT808LocationFrame = "7E020000320133077954250001000000000048000301D2C4C707376139000A00E6004F26063016235701040001900C2504000000000202000030011F31010F867E"
const sampleGB32960RealtimeFrame = "232302FE4C423941333241323152304C53313730370102ED1A070116091C0102030100000008297D161F27104C020007D0000002010104564E204E205815CA271003000A000000B40002666902800200000100A101000400FFFF001205000733444601E2B5DB06013A0F6C018E0F4001014B01054A07000000000000000000080101161F271000900001900F520F530F520F640F660F650F650F650F650F660F650F650F640F640F550F550F540F560F580F570F540F570F560F550F5A0F550F580F570F5C0F5F0F5E0F610F5E0F5D0F5E0F5F0F5E0F5D0F610F5F0F610F600F600F610F610F610F600F630F630F600F610F610F610F620F620F610F6A0F6C0F6B0F6A0F6C0F6B0F6A0F6B0F6B0F6B0F6A0F610F630F630F630F640F660F640F630F630F630F580F5A0F580F5B0F550F580F5A0F5A0F590F580F5A0F620F650F650F650F660F640F640F640F640F640F660F650F640F650F660F650F670F670F670F670F630F620F680F670F650F650F670F650F610F5F0F600F620F5E0F5C0F590F4C0F490F480F440F470F480F470F420F430F450F420F450F440F440F430F530F540F520F400F410F4109010100084B4B4B4B4A4A4B4A3001026907B2FF00FF00380002002800000008006C00016C00080008000000080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008002800070007000700070007000700070007000700070007000700070007000700070007000700070007000700070007000700080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000800080008000831010E9C0005FFFFFFFF0E9CFFFFFFFF0083320E9C1B6115E212465733FFFFFFFFFF34FFFFFFFFFF002B800009690E9C2710000D000E83002402020500002700080001000000FD00000000FFFFFFFFFFFFFFFFFFFFFFFF1FFE1FF3001DBA"
func FrameTemplate(protocol Protocol, name string) ([]byte, error) {
template := strings.ToLower(strings.TrimSpace(name))
if template == "" {
template = defaultTemplate(protocol)
}
var rawHex string
switch protocol {
case ProtocolJT808:
switch template {
case "0200", "location", "default":
rawHex = sampleJT808LocationFrame
default:
return nil, fmt.Errorf("unsupported jt808 template %q", name)
}
case ProtocolGB32960:
switch template {
case "0200", "realtime", "default":
rawHex = sampleGB32960RealtimeFrame
default:
return nil, fmt.Errorf("unsupported gb32960 template %q", name)
}
default:
return nil, fmt.Errorf("unsupported protocol %q", protocol)
}
frame, err := hex.DecodeString(rawHex)
if err != nil {
return nil, err
}
return frame, nil
}
func defaultTemplate(protocol Protocol) string {
switch protocol {
case ProtocolJT808:
return "0200"
case ProtocolGB32960:
return "realtime"
default:
return "default"
}
}