Files
lingniu-vehicle-ingest/go/vehicle-gateway/internal/gateway/tcp_server.go

748 lines
22 KiB
Go

package gateway
import (
"context"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"net"
"strings"
"sync"
"syscall"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
)
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct {
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Authenticate authentication.Authenticator
Respond FrameResponder
}
type connectionState struct {
platformName string
}
type TCPServer struct {
protocol TCPProtocol
sink eventbus.Sink
resolver identity.Resolver
logger *slog.Logger
metrics *metrics.Registry
readBufferSize int
idleTimeout time.Duration
maxConnections int
publishUnified bool
delegateFields bool
activeMu sync.Mutex
activeConns map[net.Conn]struct{}
}
type TCPServerConfig struct {
Protocol TCPProtocol
Sink eventbus.Sink
Resolver identity.Resolver
Logger *slog.Logger
Metrics *metrics.Registry
ReadBufferSize int
IdleTimeout time.Duration
MaxConnections int
PublishUnified bool
DelegateFields bool
}
const frameOperationTimeout = 30 * time.Second
var gatewayFrameDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
var gatewayResponseDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
var gatewayResponseE2ERecent = metrics.NewRecentLatencyByKey(512)
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
if cfg.Protocol.Protocol == "" {
return nil, errors.New("protocol is required")
}
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
return nil, errors.New("listen addr is required")
}
if cfg.Protocol.Extract == nil {
return nil, errors.New("frame extractor is required")
}
if cfg.Protocol.Parse == nil {
return nil, errors.New("frame parser is required")
}
if cfg.Sink == nil {
return nil, errors.New("sink is required")
}
if cfg.Logger == nil {
cfg.Logger = slog.Default()
}
if cfg.Resolver == nil {
cfg.Resolver = identity.NoopResolver{}
}
if cfg.ReadBufferSize <= 0 {
cfg.ReadBufferSize = 32 * 1024
}
if cfg.IdleTimeout <= 0 {
cfg.IdleTimeout = 2 * time.Minute
}
if cfg.MaxConnections <= 0 {
cfg.MaxConnections = 10_000
}
return &TCPServer{
protocol: cfg.Protocol,
sink: cfg.Sink,
resolver: cfg.Resolver,
logger: cfg.Logger,
metrics: cfg.Metrics,
readBufferSize: cfg.ReadBufferSize,
idleTimeout: cfg.IdleTimeout,
maxConnections: cfg.MaxConnections,
publishUnified: cfg.PublishUnified,
delegateFields: cfg.DelegateFields,
activeConns: map[net.Conn]struct{}{},
}, nil
}
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
var lc net.ListenConfig
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
if err != nil {
return err
}
defer listener.Close()
go func() {
<-ctx.Done()
_ = listener.Close()
s.closeActiveConnections()
}()
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
sem := make(chan struct{}, s.maxConnections)
var wg sync.WaitGroup
defer wg.Wait()
for {
conn, err := listener.Accept()
if err != nil {
if ctx.Err() != nil {
return nil
}
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
continue
}
select {
case sem <- struct{}{}:
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
s.handleConnection(ctx, conn)
}()
default:
s.recordConnectionRejection("max_connections")
s.recordConnectionClose("max_connections")
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
_ = conn.Close()
}
}
}
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
defer conn.Close()
s.trackConnection(conn)
defer s.untrackConnection(conn)
source := conn.RemoteAddr().String()
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
s.recordConnectionMetric(1)
defer s.recordConnectionMetric(-1)
log.Debug("tcp connection opened")
defer log.Debug("tcp connection closed")
readBuffer := make([]byte, s.readBufferSize)
var pending []byte
state := &connectionState{}
for {
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
n, err := conn.Read(readBuffer)
if n > 0 {
pending = append(pending, readBuffer[:n]...)
frames, remainder, extractErr := s.protocol.Extract(pending)
if extractErr != nil {
log.Warn("frame extraction failed", "error", extractErr)
s.recordConnectionClose("extract_error")
return
}
pending = remainder
for _, frame := range frames {
s.handleFrame(ctx, conn, frame, source, state)
}
}
if err != nil {
if ctx.Err() != nil {
s.recordConnectionClose("context_cancelled")
return
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
log.Debug("tcp connection idle timeout")
s.recordConnectionClose("read_timeout")
return
}
if isRoutineTCPReadClose(err) {
log.Debug("tcp connection closed by peer", "error", err)
s.recordConnectionClose("remote_closed")
return
}
log.Warn("tcp read failed", "error", err)
s.recordConnectionClose("read_error")
return
}
if ctx.Err() != nil {
s.recordConnectionClose("context_cancelled")
return
}
}
}
func isRoutineTCPReadClose(err error) bool {
if err == nil {
return false
}
if errors.Is(err, io.EOF) ||
errors.Is(err, net.ErrClosed) ||
errors.Is(err, syscall.ECONNRESET) ||
errors.Is(err, syscall.EPIPE) {
return true
}
text := strings.ToLower(err.Error())
return strings.Contains(text, "connection reset by peer") ||
strings.Contains(text, "broken pipe") ||
strings.Contains(text, "use of closed network connection")
}
func (s *TCPServer) trackConnection(conn net.Conn) {
if conn == nil {
return
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
if s.activeConns == nil {
s.activeConns = map[net.Conn]struct{}{}
}
s.activeConns[conn] = struct{}{}
}
func (s *TCPServer) untrackConnection(conn net.Conn) {
if conn == nil {
return
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
delete(s.activeConns, conn)
}
func (s *TCPServer) closeActiveConnections() {
s.activeMu.Lock()
conns := make([]net.Conn, 0, len(s.activeConns))
for conn := range s.activeConns {
conns = append(conns, conn)
}
s.activeMu.Unlock()
for _, conn := range conns {
_ = conn.Close()
}
}
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) {
started := time.Now()
frameStatus := envelope.ParseBadFrame
defer func() {
s.recordFrameDuration(frameStatus, time.Since(started))
}()
frameCtx, cancelFrame := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
defer cancelFrame()
receivedAtMS := time.Now().UnixMilli()
env, err := s.protocol.Parse(raw, receivedAtMS, source)
if err != nil {
env = envelope.FrameEnvelope{
Protocol: s.protocol.Protocol,
SourceEndpoint: source,
ReceivedAtMS: receivedAtMS,
EventTimeMS: receivedAtMS,
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
ParseStatus: envelope.ParseBadFrame,
ParseError: err.Error(),
}
env.EventID = env.StableEventID()
s.recordParseErrorMetric(err)
} else {
if s.protocol.Authenticate != nil {
result := s.protocol.Authenticate.Authenticate(env)
authentication.Apply(&env, result)
if result.Applicable {
s.recordAuthenticationMetric(result)
}
}
authentication.RedactParsedCredentials(&env)
if envelope.RequiresVehicleIdentity(env) {
resolveStarted := time.Now()
resolved, resolveErr := s.resolver.Resolve(frameCtx, env)
if resolveErr != nil {
if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" {
env = resolved
}
identityStatus := identityErrorStatus(resolveErr)
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
annotateIdentityError(&env, resolveErr)
env.ParseStatus = envelope.ParsePartial
s.recordIdentityMetric(identityStatus)
s.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr))
} else {
env = resolved
annotateIdentityUnresolved(&env)
identityStatus := identityStatus(env)
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
s.recordIdentityMetric(identityStatus)
if identityStatus != "resolved" {
s.recordIdentityIssueMetric(identityStatus, env, "no_binding")
}
recordGatewayIdentityCacheStatus(s.metrics, s.protocol.Protocol, env)
}
} else {
s.recordIdentitySkipMetric("non_vehicle_frame")
}
}
enrichConnectionPlatform(&env, state)
frameStatus = env.ParseStatus
env.EventKind = envelope.EventKindRaw
s.recordFrameMetric(env.ParseStatus)
if env.ParseStatus != envelope.ParseBadFrame {
realtime.EnsureParsedFields(&env)
}
canonicalRaw := canonicalRawEnvelope(env)
if err := s.sink.PublishRaw(frameCtx, canonicalRaw); err != nil {
s.recordPublishMetric("raw", "error")
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
s.recordPublishMetric("raw", "ok")
if env.ParseStatus == envelope.ParseBadFrame {
return
}
s.writeProtocolResponse(conn, raw, env, started)
if shouldCloseAfterAuthentication(env) && conn != nil {
_ = conn.Close()
return
}
if s.delegateFields {
fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env)
s.recordFieldsMetric(fieldsStatus)
if ok {
s.recordFieldsCount(fieldsStatus, fieldCount)
s.recordPublishMetric("fields", "delegated")
}
} else {
fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env)
if !ok {
s.recordFieldsMetric(fieldsStatus)
} else {
if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil {
s.recordFieldsMetric(gatewayFieldsPublishError)
s.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields))
s.recordPublishMetric("fields", "error")
s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
} else {
s.recordFieldsMetric(fieldsStatus)
s.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields))
s.recordPublishMetric("fields", "ok")
}
}
}
if s.publishUnified {
if err := s.sink.PublishUnified(frameCtx, canonicalRaw); err != nil {
s.recordPublishMetric("unified", "error")
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
} else {
s.recordPublishMetric("unified", "ok")
}
}
}
func (s *TCPServer) writeProtocolResponse(conn net.Conn, raw []byte, env envelope.FrameEnvelope, frameStarted time.Time) {
if s.protocol.Respond == nil {
return
}
status := "skipped"
defer func() {
s.recordResponseDuration(env.MessageID, status, time.Since(frameStarted))
}()
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
status = "build_error"
s.recordResponseMetric(env.MessageID, "build_error")
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
s.recordResponseMetric(env.MessageID, "skipped")
return
}
if conn == nil {
status = "write_error"
s.recordResponseMetric(env.MessageID, "write_error")
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", "nil connection")
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
status = "write_error"
s.recordResponseMetric(env.MessageID, "write_error")
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
status = "ok"
s.recordResponseMetric(env.MessageID, "ok")
}
func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionState) {
if env == nil || state == nil || env.ParseStatus == envelope.ParseBadFrame {
return
}
if env.Fields == nil {
env.Fields = map[string]any{}
}
if shouldCloseAfterAuthentication(*env) {
return
}
if current := strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])); current != "" && current != "<nil>" {
state.platformName = current
}
if state.platformName == "" {
return
}
if strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])) == "" || fmt.Sprint(env.Fields["platform_account"]) == "<nil>" {
env.Fields["platform_account"] = state.platformName
}
if strings.TrimSpace(env.PlatformName) == "" {
env.PlatformName = state.platformName
}
if strings.TrimSpace(env.SourceCode) == "" {
env.SourceCode = sourceCodeFromPlatformName(state.platformName)
}
if strings.TrimSpace(env.SourceKind) == "" || strings.EqualFold(strings.TrimSpace(env.SourceKind), "UNKNOWN") {
env.SourceKind = "PLATFORM"
}
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
if _, ok := env.Parsed["platform_name"]; !ok {
env.Parsed["platform_name"] = state.platformName
}
}
func sourceCodeFromPlatformName(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
var builder strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
builder.WriteRune(r)
case r >= 'A' && r <= 'Z':
builder.WriteRune(r)
case r >= '0' && r <= '9':
builder.WriteRune(r)
case r == '_' || r == '-' || r == '.':
builder.WriteRune(r)
}
}
return builder.String()
}
func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) {
if s.metrics == nil {
return
}
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": string(status),
}
s.metrics.IncCounter("vehicle_gateway_frames_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_frame_unix_seconds", labels)
}
func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
if s.metrics == nil {
return
}
elapsedMS := float64(elapsed.Milliseconds())
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": string(status),
}
s.metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": string(status),
}, elapsedMS)
s.metrics.ObserveHistogram("vehicle_gateway_frame_duration_ms_histogram", labels, gatewayFrameDurationBucketsMS, elapsedMS)
}
func (s *TCPServer) recordPublishMetric(kind string, status string) {
if s.metrics == nil {
return
}
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"kind": kind,
"status": status,
}
s.metrics.IncCounter("vehicle_gateway_publish_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_publish_unix_seconds", labels)
}
func (s *TCPServer) recordFieldsMetric(status string) {
recordGatewayFieldsMetric(s.metrics, s.protocol.Protocol, status)
}
func (s *TCPServer) recordFieldsCount(status string, fieldCount int) {
recordGatewayFieldsCount(s.metrics, s.protocol.Protocol, status, fieldCount)
}
func (s *TCPServer) recordResponseMetric(messageID string, status string) {
if s.metrics == nil {
return
}
messageID = strings.TrimSpace(messageID)
if messageID == "" {
messageID = "unknown"
}
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"message_id": messageID,
"status": status,
}
s.metrics.IncCounter("vehicle_gateway_response_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_response_unix_seconds", labels)
}
func (s *TCPServer) recordResponseDuration(messageID string, status string, elapsed time.Duration) {
if s.metrics == nil {
return
}
messageID = strings.TrimSpace(messageID)
if messageID == "" {
messageID = "unknown"
}
elapsedMS := float64(elapsed.Microseconds()) / 1000
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"message_id": messageID,
"status": status,
}
s.metrics.ObserveHistogram("vehicle_gateway_response_duration_ms_histogram", labels, gatewayResponseDurationBucketsMS, elapsedMS)
if status != "ok" {
return
}
protocol := string(s.protocol.Protocol)
p99, samples := gatewayResponseE2ERecent.Observe(protocol, elapsedMS)
protocolLabels := metrics.Labels{"protocol": protocol}
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_p99_ms", protocolLabels, p99)
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_samples", protocolLabels, float64(samples))
}
func (s *TCPServer) recordAuthenticationMetric(result authentication.Result) {
if s.metrics == nil || !result.Applicable {
return
}
s.metrics.IncCounter("vehicle_gateway_authentication_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"mode": string(result.Mode),
"source": result.Source,
"status": result.Status,
})
}
func (s *TCPServer) recordParseErrorMetric(err error) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": classifyError(err),
})
}
func (s *TCPServer) recordIdentityMetric(status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": status,
})
}
func (s *TCPServer) recordIdentitySkipMetric(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func (s *TCPServer) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) {
if s.metrics == nil {
return
}
messageID := strings.TrimSpace(env.MessageID)
if messageID == "" {
messageID = "unknown"
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "unknown"
}
s.metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": status,
"message_id": messageID,
"reason": reason,
})
}
func (s *TCPServer) recordIdentityDuration(status string, elapsed time.Duration) {
recordGatewayIdentityDuration(s.metrics, s.protocol.Protocol, status, elapsed)
}
func (s *TCPServer) recordConnectionMetric(delta float64) {
if s.metrics == nil {
return
}
s.metrics.AddGauge("vehicle_gateway_active_connections", metrics.Labels{
"protocol": string(s.protocol.Protocol),
}, delta)
}
func (s *TCPServer) recordConnectionRejection(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_connection_rejections_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func (s *TCPServer) recordConnectionClose(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func shouldCloseAfterAuthentication(env envelope.FrameEnvelope) bool {
return env.AuthenticationEnforced &&
strings.TrimSpace(env.AuthenticationStatus) != "" &&
env.AuthenticationStatus != authentication.StatusAccepted
}
func identityStatus(env envelope.FrameEnvelope) string {
if strings.TrimSpace(env.VIN) != "" {
return "resolved"
}
if env.ParseStatus == envelope.ParsePartial {
return "error"
}
return "unresolved"
}
func identityIssueReason(status string, err error) string {
switch status {
case "timeout":
return "timeout"
case "error":
if err == nil {
return "resolver_error"
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
switch {
case strings.Contains(text, "connection"), strings.Contains(text, "tcp"), strings.Contains(text, "network"):
return "identity_store_connection"
case strings.Contains(text, "timeout"), strings.Contains(text, "deadline"):
return "timeout"
default:
return "resolver_error"
}
default:
return "no_binding"
}
}
func annotateIdentityUnresolved(env *envelope.FrameEnvelope) {
if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" {
return
}
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
if _, exists := env.Parsed["identity"]; exists {
return
}
env.Parsed["identity"] = map[string]any{
"resolved": false,
"reason": "no_binding",
}
}
func classifyError(err error) string {
text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err)))
switch {
case text == "":
return "unknown"
case strings.Contains(text, "bcc") || strings.Contains(text, "checksum"):
return "checksum"
case strings.Contains(text, "short") || strings.Contains(text, "truncated") || strings.Contains(text, "length"):
return "length"
case strings.Contains(text, "json"):
return "json"
case strings.Contains(text, "start"):
return "start_symbol"
default:
return "parse"
}
}
func (p TCPProtocol) String() string {
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
}