package gateway import ( "context" "encoding/hex" "errors" "fmt" "io" "log/slog" "net" "strings" "sync" "syscall" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" ) type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error) type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error) type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error) type TCPProtocol struct { Protocol envelope.Protocol Addr string Extract FrameExtractor Parse FrameParser Authenticate authentication.Authenticator Respond FrameResponder } type connectionState struct { platformName string } type TCPServer struct { protocol TCPProtocol sink eventbus.Sink resolver identity.Resolver logger *slog.Logger metrics *metrics.Registry readBufferSize int idleTimeout time.Duration maxConnections int publishUnified bool delegateFields bool activeMu sync.Mutex activeConns map[net.Conn]struct{} } type TCPServerConfig struct { Protocol TCPProtocol Sink eventbus.Sink Resolver identity.Resolver Logger *slog.Logger Metrics *metrics.Registry ReadBufferSize int IdleTimeout time.Duration MaxConnections int PublishUnified bool DelegateFields bool } const frameOperationTimeout = 30 * time.Second var gatewayFrameDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} var gatewayResponseDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000} var gatewayResponseE2ERecent = metrics.NewRecentLatencyByKey(512) func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) { if cfg.Protocol.Protocol == "" { return nil, errors.New("protocol is required") } if strings.TrimSpace(cfg.Protocol.Addr) == "" { return nil, errors.New("listen addr is required") } if cfg.Protocol.Extract == nil { return nil, errors.New("frame extractor is required") } if cfg.Protocol.Parse == nil { return nil, errors.New("frame parser is required") } if cfg.Sink == nil { return nil, errors.New("sink is required") } if cfg.Logger == nil { cfg.Logger = slog.Default() } if cfg.Resolver == nil { cfg.Resolver = identity.NoopResolver{} } if cfg.ReadBufferSize <= 0 { cfg.ReadBufferSize = 32 * 1024 } if cfg.IdleTimeout <= 0 { cfg.IdleTimeout = 2 * time.Minute } if cfg.MaxConnections <= 0 { cfg.MaxConnections = 10_000 } return &TCPServer{ protocol: cfg.Protocol, sink: cfg.Sink, resolver: cfg.Resolver, logger: cfg.Logger, metrics: cfg.Metrics, readBufferSize: cfg.ReadBufferSize, idleTimeout: cfg.IdleTimeout, maxConnections: cfg.MaxConnections, publishUnified: cfg.PublishUnified, delegateFields: cfg.DelegateFields, activeConns: map[net.Conn]struct{}{}, }, nil } func (s *TCPServer) ListenAndServe(ctx context.Context) error { var lc net.ListenConfig listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr) if err != nil { return err } defer listener.Close() go func() { <-ctx.Done() _ = listener.Close() s.closeActiveConnections() }() s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String()) sem := make(chan struct{}, s.maxConnections) var wg sync.WaitGroup defer wg.Wait() for { conn, err := listener.Accept() if err != nil { if ctx.Err() != nil { return nil } s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err) continue } select { case sem <- struct{}{}: wg.Add(1) go func() { defer wg.Done() defer func() { <-sem }() s.handleConnection(ctx, conn) }() default: s.recordConnectionRejection("max_connections") s.recordConnectionClose("max_connections") s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String()) _ = conn.Close() } } } func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) { defer conn.Close() s.trackConnection(conn) defer s.untrackConnection(conn) source := conn.RemoteAddr().String() log := s.logger.With("protocol", s.protocol.Protocol, "remote", source) s.recordConnectionMetric(1) defer s.recordConnectionMetric(-1) log.Debug("tcp connection opened") defer log.Debug("tcp connection closed") readBuffer := make([]byte, s.readBufferSize) var pending []byte state := &connectionState{} for { _ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout)) n, err := conn.Read(readBuffer) if n > 0 { pending = append(pending, readBuffer[:n]...) frames, remainder, extractErr := s.protocol.Extract(pending) if extractErr != nil { log.Warn("frame extraction failed", "error", extractErr) s.recordConnectionClose("extract_error") return } pending = remainder for _, frame := range frames { s.handleFrame(ctx, conn, frame, source, state) } } if err != nil { if ctx.Err() != nil { s.recordConnectionClose("context_cancelled") return } var netErr net.Error if errors.As(err, &netErr) && netErr.Timeout() { log.Debug("tcp connection idle timeout") s.recordConnectionClose("read_timeout") return } if isRoutineTCPReadClose(err) { log.Debug("tcp connection closed by peer", "error", err) s.recordConnectionClose("remote_closed") return } log.Warn("tcp read failed", "error", err) s.recordConnectionClose("read_error") return } if ctx.Err() != nil { s.recordConnectionClose("context_cancelled") return } } } func isRoutineTCPReadClose(err error) bool { if err == nil { return false } if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) { return true } text := strings.ToLower(err.Error()) return strings.Contains(text, "connection reset by peer") || strings.Contains(text, "broken pipe") || strings.Contains(text, "use of closed network connection") } func (s *TCPServer) trackConnection(conn net.Conn) { if conn == nil { return } s.activeMu.Lock() defer s.activeMu.Unlock() if s.activeConns == nil { s.activeConns = map[net.Conn]struct{}{} } s.activeConns[conn] = struct{}{} } func (s *TCPServer) untrackConnection(conn net.Conn) { if conn == nil { return } s.activeMu.Lock() defer s.activeMu.Unlock() delete(s.activeConns, conn) } func (s *TCPServer) closeActiveConnections() { s.activeMu.Lock() conns := make([]net.Conn, 0, len(s.activeConns)) for conn := range s.activeConns { conns = append(conns, conn) } s.activeMu.Unlock() for _, conn := range conns { _ = conn.Close() } } func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) { started := time.Now() frameStatus := envelope.ParseBadFrame defer func() { s.recordFrameDuration(frameStatus, time.Since(started)) }() frameCtx, cancelFrame := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout) defer cancelFrame() receivedAtMS := time.Now().UnixMilli() env, err := s.protocol.Parse(raw, receivedAtMS, source) if err != nil { env = envelope.FrameEnvelope{ Protocol: s.protocol.Protocol, SourceEndpoint: source, ReceivedAtMS: receivedAtMS, EventTimeMS: receivedAtMS, RawHex: strings.ToUpper(hex.EncodeToString(raw)), ParseStatus: envelope.ParseBadFrame, ParseError: err.Error(), } env.EventID = env.StableEventID() s.recordParseErrorMetric(err) } else { if s.protocol.Authenticate != nil { result := s.protocol.Authenticate.Authenticate(env) authentication.Apply(&env, result) if result.Applicable { s.recordAuthenticationMetric(result) } } authentication.RedactParsedCredentials(&env) if envelope.RequiresVehicleIdentity(env) { resolveStarted := time.Now() resolved, resolveErr := s.resolver.Resolve(frameCtx, env) if resolveErr != nil { if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" { env = resolved } identityStatus := identityErrorStatus(resolveErr) s.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr) annotateIdentityError(&env, resolveErr) env.ParseStatus = envelope.ParsePartial s.recordIdentityMetric(identityStatus) s.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr)) } else { env = resolved annotateIdentityUnresolved(&env) identityStatus := identityStatus(env) s.recordIdentityDuration(identityStatus, time.Since(resolveStarted)) s.recordIdentityMetric(identityStatus) if identityStatus != "resolved" { s.recordIdentityIssueMetric(identityStatus, env, "no_binding") } recordGatewayIdentityCacheStatus(s.metrics, s.protocol.Protocol, env) } } else { s.recordIdentitySkipMetric("non_vehicle_frame") } } enrichConnectionPlatform(&env, state) frameStatus = env.ParseStatus env.EventKind = envelope.EventKindRaw s.recordFrameMetric(env.ParseStatus) if env.ParseStatus != envelope.ParseBadFrame { realtime.EnsureParsedFields(&env) } canonicalRaw := canonicalRawEnvelope(env) if err := s.sink.PublishRaw(frameCtx, canonicalRaw); err != nil { s.recordPublishMetric("raw", "error") s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) return } s.recordPublishMetric("raw", "ok") if env.ParseStatus == envelope.ParseBadFrame { return } s.writeProtocolResponse(conn, raw, env, started) if shouldCloseAfterAuthentication(env) && conn != nil { _ = conn.Close() return } if s.delegateFields { fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env) s.recordFieldsMetric(fieldsStatus) if ok { s.recordFieldsCount(fieldsStatus, fieldCount) s.recordPublishMetric("fields", "delegated") } } else { fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env) if !ok { s.recordFieldsMetric(fieldsStatus) } else { if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil { s.recordFieldsMetric(gatewayFieldsPublishError) s.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields)) s.recordPublishMetric("fields", "error") s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) } else { s.recordFieldsMetric(fieldsStatus) s.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields)) s.recordPublishMetric("fields", "ok") } } } if s.publishUnified { if err := s.sink.PublishUnified(frameCtx, canonicalRaw); err != nil { s.recordPublishMetric("unified", "error") s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) } else { s.recordPublishMetric("unified", "ok") } } } func (s *TCPServer) writeProtocolResponse(conn net.Conn, raw []byte, env envelope.FrameEnvelope, frameStarted time.Time) { if s.protocol.Respond == nil { return } status := "skipped" defer func() { s.recordResponseDuration(env.MessageID, status, time.Since(frameStarted)) }() response, ok, err := s.protocol.Respond(raw, env) if err != nil { status = "build_error" s.recordResponseMetric(env.MessageID, "build_error") s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) return } if !ok || len(response) == 0 { s.recordResponseMetric(env.MessageID, "skipped") return } if conn == nil { status = "write_error" s.recordResponseMetric(env.MessageID, "write_error") s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", "nil connection") return } _ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second)) if _, err := conn.Write(response); err != nil { status = "write_error" s.recordResponseMetric(env.MessageID, "write_error") s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err) return } status = "ok" s.recordResponseMetric(env.MessageID, "ok") } func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionState) { if env == nil || state == nil || env.ParseStatus == envelope.ParseBadFrame { return } if env.Fields == nil { env.Fields = map[string]any{} } if shouldCloseAfterAuthentication(*env) { return } if current := strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])); current != "" && current != "" { state.platformName = current } if state.platformName == "" { return } if strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])) == "" || fmt.Sprint(env.Fields["platform_account"]) == "" { env.Fields["platform_account"] = state.platformName } if strings.TrimSpace(env.PlatformName) == "" { env.PlatformName = state.platformName } if strings.TrimSpace(env.SourceCode) == "" { env.SourceCode = sourceCodeFromPlatformName(state.platformName) } if strings.TrimSpace(env.SourceKind) == "" || strings.EqualFold(strings.TrimSpace(env.SourceKind), "UNKNOWN") { env.SourceKind = "PLATFORM" } if env.Parsed == nil { env.Parsed = map[string]any{} } if _, ok := env.Parsed["platform_name"]; !ok { env.Parsed["platform_name"] = state.platformName } } func sourceCodeFromPlatformName(value string) string { value = strings.TrimSpace(value) if value == "" { return "" } var builder strings.Builder for _, r := range value { switch { case r >= 'a' && r <= 'z': builder.WriteRune(r) case r >= 'A' && r <= 'Z': builder.WriteRune(r) case r >= '0' && r <= '9': builder.WriteRune(r) case r == '_' || r == '-' || r == '.': builder.WriteRune(r) } } return builder.String() } func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) { if s.metrics == nil { return } labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": string(status), } s.metrics.IncCounter("vehicle_gateway_frames_total", labels) metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_frame_unix_seconds", labels) } func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) { if s.metrics == nil { return } elapsedMS := float64(elapsed.Milliseconds()) labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": string(status), } s.metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": string(status), }, elapsedMS) s.metrics.ObserveHistogram("vehicle_gateway_frame_duration_ms_histogram", labels, gatewayFrameDurationBucketsMS, elapsedMS) } func (s *TCPServer) recordPublishMetric(kind string, status string) { if s.metrics == nil { return } labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "kind": kind, "status": status, } s.metrics.IncCounter("vehicle_gateway_publish_total", labels) metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_publish_unix_seconds", labels) } func (s *TCPServer) recordFieldsMetric(status string) { recordGatewayFieldsMetric(s.metrics, s.protocol.Protocol, status) } func (s *TCPServer) recordFieldsCount(status string, fieldCount int) { recordGatewayFieldsCount(s.metrics, s.protocol.Protocol, status, fieldCount) } func (s *TCPServer) recordResponseMetric(messageID string, status string) { if s.metrics == nil { return } messageID = strings.TrimSpace(messageID) if messageID == "" { messageID = "unknown" } labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "message_id": messageID, "status": status, } s.metrics.IncCounter("vehicle_gateway_response_total", labels) metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_response_unix_seconds", labels) } func (s *TCPServer) recordResponseDuration(messageID string, status string, elapsed time.Duration) { if s.metrics == nil { return } messageID = strings.TrimSpace(messageID) if messageID == "" { messageID = "unknown" } elapsedMS := float64(elapsed.Microseconds()) / 1000 labels := metrics.Labels{ "protocol": string(s.protocol.Protocol), "message_id": messageID, "status": status, } s.metrics.ObserveHistogram("vehicle_gateway_response_duration_ms_histogram", labels, gatewayResponseDurationBucketsMS, elapsedMS) if status != "ok" { return } protocol := string(s.protocol.Protocol) p99, samples := gatewayResponseE2ERecent.Observe(protocol, elapsedMS) protocolLabels := metrics.Labels{"protocol": protocol} s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_p99_ms", protocolLabels, p99) s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_samples", protocolLabels, float64(samples)) } func (s *TCPServer) recordAuthenticationMetric(result authentication.Result) { if s.metrics == nil || !result.Applicable { return } s.metrics.IncCounter("vehicle_gateway_authentication_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "mode": string(result.Mode), "source": result.Source, "status": result.Status, }) } func (s *TCPServer) recordParseErrorMetric(err error) { if s.metrics == nil { return } s.metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "reason": classifyError(err), }) } func (s *TCPServer) recordIdentityMetric(status string) { if s.metrics == nil { return } s.metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": status, }) } func (s *TCPServer) recordIdentitySkipMetric(reason string) { if s.metrics == nil { return } s.metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "reason": reason, }) } func (s *TCPServer) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) { if s.metrics == nil { return } messageID := strings.TrimSpace(env.MessageID) if messageID == "" { messageID = "unknown" } reason = strings.TrimSpace(reason) if reason == "" { reason = "unknown" } s.metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "status": status, "message_id": messageID, "reason": reason, }) } func (s *TCPServer) recordIdentityDuration(status string, elapsed time.Duration) { recordGatewayIdentityDuration(s.metrics, s.protocol.Protocol, status, elapsed) } func (s *TCPServer) recordConnectionMetric(delta float64) { if s.metrics == nil { return } s.metrics.AddGauge("vehicle_gateway_active_connections", metrics.Labels{ "protocol": string(s.protocol.Protocol), }, delta) } func (s *TCPServer) recordConnectionRejection(reason string) { if s.metrics == nil { return } s.metrics.IncCounter("vehicle_gateway_connection_rejections_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "reason": reason, }) } func (s *TCPServer) recordConnectionClose(reason string) { if s.metrics == nil { return } s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{ "protocol": string(s.protocol.Protocol), "reason": reason, }) } func shouldCloseAfterAuthentication(env envelope.FrameEnvelope) bool { return env.AuthenticationEnforced && strings.TrimSpace(env.AuthenticationStatus) != "" && env.AuthenticationStatus != authentication.StatusAccepted } func identityStatus(env envelope.FrameEnvelope) string { if strings.TrimSpace(env.VIN) != "" { return "resolved" } if env.ParseStatus == envelope.ParsePartial { return "error" } return "unresolved" } func identityIssueReason(status string, err error) string { switch status { case "timeout": return "timeout" case "error": if err == nil { return "resolver_error" } text := strings.ToLower(strings.TrimSpace(err.Error())) switch { case strings.Contains(text, "connection"), strings.Contains(text, "tcp"), strings.Contains(text, "network"): return "identity_store_connection" case strings.Contains(text, "timeout"), strings.Contains(text, "deadline"): return "timeout" default: return "resolver_error" } default: return "no_binding" } } func annotateIdentityUnresolved(env *envelope.FrameEnvelope) { if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" { return } if env.Parsed == nil { env.Parsed = map[string]any{} } if _, exists := env.Parsed["identity"]; exists { return } env.Parsed["identity"] = map[string]any{ "resolved": false, "reason": "no_binding", } } func classifyError(err error) string { text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err))) switch { case text == "": return "unknown" case strings.Contains(text, "bcc") || strings.Contains(text, "checksum"): return "checksum" case strings.Contains(text, "short") || strings.Contains(text, "truncated") || strings.Contains(text, "length"): return "length" case strings.Contains(text, "json"): return "json" case strings.Contains(text, "start"): return "start_symbol" default: return "parse" } } func (p TCPProtocol) String() string { return fmt.Sprintf("%s@%s", p.Protocol, p.Addr) }