Add monitoring metrics, numeric sorting, report-time display, vehicle number labels, and map layout fixes. Include accumulated mileage, hydrogen, OneOS authentication, open platform, ingestion, deployment, and UI updates. Exclude temporary files, generated reports, and credentials. Validate frontend 632 tests and production build, API packages, vehicle-map tests, and release installer tests. Gateway suite has one pre-existing date-sensitive identity throttling test failure reproduced on the parent commit.
761 lines
22 KiB
Go
761 lines
22 KiB
Go
package gateway
|
|
|
|
import (
|
|
"context"
|
|
"encoding/hex"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"net"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"time"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
|
)
|
|
|
|
type FrameExtractor func([]byte) (frames [][]byte, remainder []byte, err error)
|
|
|
|
type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (envelope.FrameEnvelope, error)
|
|
|
|
type PlatformFrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint, platformAccount string) (envelope.FrameEnvelope, error)
|
|
|
|
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
|
|
|
|
type TCPProtocol struct {
|
|
Protocol envelope.Protocol
|
|
Addr string
|
|
Extract FrameExtractor
|
|
Parse FrameParser
|
|
ParseForPlatform PlatformFrameParser
|
|
Authenticate authentication.Authenticator
|
|
Respond FrameResponder
|
|
}
|
|
|
|
type connectionState struct {
|
|
platformName string
|
|
}
|
|
|
|
type TCPServer struct {
|
|
protocol TCPProtocol
|
|
sink eventbus.Sink
|
|
resolver identity.Resolver
|
|
logger *slog.Logger
|
|
metrics *metrics.Registry
|
|
readBufferSize int
|
|
idleTimeout time.Duration
|
|
maxConnections int
|
|
publishUnified bool
|
|
delegateFields bool
|
|
activeMu sync.Mutex
|
|
activeConns map[net.Conn]struct{}
|
|
}
|
|
|
|
type TCPServerConfig struct {
|
|
Protocol TCPProtocol
|
|
Sink eventbus.Sink
|
|
Resolver identity.Resolver
|
|
Logger *slog.Logger
|
|
Metrics *metrics.Registry
|
|
ReadBufferSize int
|
|
IdleTimeout time.Duration
|
|
MaxConnections int
|
|
PublishUnified bool
|
|
DelegateFields bool
|
|
}
|
|
|
|
const frameOperationTimeout = 30 * time.Second
|
|
|
|
var gatewayFrameDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
|
|
var gatewayResponseDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
|
|
var gatewayResponseE2ERecent = metrics.NewRecentLatencyByKey(512)
|
|
|
|
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
|
|
if cfg.Protocol.Protocol == "" {
|
|
return nil, errors.New("protocol is required")
|
|
}
|
|
if strings.TrimSpace(cfg.Protocol.Addr) == "" {
|
|
return nil, errors.New("listen addr is required")
|
|
}
|
|
if cfg.Protocol.Extract == nil {
|
|
return nil, errors.New("frame extractor is required")
|
|
}
|
|
if cfg.Protocol.Parse == nil {
|
|
return nil, errors.New("frame parser is required")
|
|
}
|
|
if cfg.Sink == nil {
|
|
return nil, errors.New("sink is required")
|
|
}
|
|
if cfg.Logger == nil {
|
|
cfg.Logger = slog.Default()
|
|
}
|
|
if cfg.Resolver == nil {
|
|
cfg.Resolver = identity.NoopResolver{}
|
|
}
|
|
if cfg.ReadBufferSize <= 0 {
|
|
cfg.ReadBufferSize = 32 * 1024
|
|
}
|
|
if cfg.IdleTimeout <= 0 {
|
|
cfg.IdleTimeout = 2 * time.Minute
|
|
}
|
|
if cfg.MaxConnections <= 0 {
|
|
cfg.MaxConnections = 10_000
|
|
}
|
|
return &TCPServer{
|
|
protocol: cfg.Protocol,
|
|
sink: cfg.Sink,
|
|
resolver: cfg.Resolver,
|
|
logger: cfg.Logger,
|
|
metrics: cfg.Metrics,
|
|
readBufferSize: cfg.ReadBufferSize,
|
|
idleTimeout: cfg.IdleTimeout,
|
|
maxConnections: cfg.MaxConnections,
|
|
publishUnified: cfg.PublishUnified,
|
|
delegateFields: cfg.DelegateFields,
|
|
activeConns: map[net.Conn]struct{}{},
|
|
}, nil
|
|
}
|
|
|
|
func (s *TCPServer) ListenAndServe(ctx context.Context) error {
|
|
var lc net.ListenConfig
|
|
listener, err := lc.Listen(ctx, "tcp", s.protocol.Addr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer listener.Close()
|
|
|
|
go func() {
|
|
<-ctx.Done()
|
|
_ = listener.Close()
|
|
s.closeActiveConnections()
|
|
}()
|
|
|
|
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
|
|
sem := make(chan struct{}, s.maxConnections)
|
|
var wg sync.WaitGroup
|
|
defer wg.Wait()
|
|
|
|
for {
|
|
conn, err := listener.Accept()
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return nil
|
|
}
|
|
s.logger.Warn("tcp accept failed", "protocol", s.protocol.Protocol, "error", err)
|
|
continue
|
|
}
|
|
select {
|
|
case sem <- struct{}{}:
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
defer func() { <-sem }()
|
|
s.handleConnection(ctx, conn)
|
|
}()
|
|
default:
|
|
s.recordConnectionRejection("max_connections")
|
|
s.recordConnectionClose("max_connections")
|
|
s.logger.Warn("tcp connection rejected: max connections reached", "protocol", s.protocol.Protocol, "remote", conn.RemoteAddr().String())
|
|
_ = conn.Close()
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
|
|
defer conn.Close()
|
|
s.trackConnection(conn)
|
|
defer s.untrackConnection(conn)
|
|
|
|
source := conn.RemoteAddr().String()
|
|
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
|
|
s.recordConnectionMetric(1)
|
|
defer s.recordConnectionMetric(-1)
|
|
log.Debug("tcp connection opened")
|
|
defer log.Debug("tcp connection closed")
|
|
|
|
readBuffer := make([]byte, s.readBufferSize)
|
|
var pending []byte
|
|
state := &connectionState{}
|
|
for {
|
|
_ = conn.SetReadDeadline(time.Now().Add(s.idleTimeout))
|
|
n, err := conn.Read(readBuffer)
|
|
if n > 0 {
|
|
pending = append(pending, readBuffer[:n]...)
|
|
frames, remainder, extractErr := s.protocol.Extract(pending)
|
|
if extractErr != nil {
|
|
log.Warn("frame extraction failed", "error", extractErr)
|
|
s.recordConnectionClose("extract_error")
|
|
return
|
|
}
|
|
pending = remainder
|
|
for _, frame := range frames {
|
|
s.handleFrame(ctx, conn, frame, source, state)
|
|
}
|
|
}
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
s.recordConnectionClose("context_cancelled")
|
|
return
|
|
}
|
|
var netErr net.Error
|
|
if errors.As(err, &netErr) && netErr.Timeout() {
|
|
log.Debug("tcp connection idle timeout")
|
|
s.recordConnectionClose("read_timeout")
|
|
return
|
|
}
|
|
if isRoutineTCPReadClose(err) {
|
|
log.Debug("tcp connection closed by peer", "error", err)
|
|
s.recordConnectionClose("remote_closed")
|
|
return
|
|
}
|
|
log.Warn("tcp read failed", "error", err)
|
|
s.recordConnectionClose("read_error")
|
|
return
|
|
}
|
|
if ctx.Err() != nil {
|
|
s.recordConnectionClose("context_cancelled")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
func isRoutineTCPReadClose(err error) bool {
|
|
if err == nil {
|
|
return false
|
|
}
|
|
if errors.Is(err, io.EOF) ||
|
|
errors.Is(err, net.ErrClosed) ||
|
|
errors.Is(err, syscall.ECONNRESET) ||
|
|
errors.Is(err, syscall.EPIPE) {
|
|
return true
|
|
}
|
|
text := strings.ToLower(err.Error())
|
|
return strings.Contains(text, "connection reset by peer") ||
|
|
strings.Contains(text, "broken pipe") ||
|
|
strings.Contains(text, "use of closed network connection")
|
|
}
|
|
|
|
func (s *TCPServer) trackConnection(conn net.Conn) {
|
|
if conn == nil {
|
|
return
|
|
}
|
|
s.activeMu.Lock()
|
|
defer s.activeMu.Unlock()
|
|
if s.activeConns == nil {
|
|
s.activeConns = map[net.Conn]struct{}{}
|
|
}
|
|
s.activeConns[conn] = struct{}{}
|
|
}
|
|
|
|
func (s *TCPServer) untrackConnection(conn net.Conn) {
|
|
if conn == nil {
|
|
return
|
|
}
|
|
s.activeMu.Lock()
|
|
defer s.activeMu.Unlock()
|
|
delete(s.activeConns, conn)
|
|
}
|
|
|
|
func (s *TCPServer) closeActiveConnections() {
|
|
s.activeMu.Lock()
|
|
conns := make([]net.Conn, 0, len(s.activeConns))
|
|
for conn := range s.activeConns {
|
|
conns = append(conns, conn)
|
|
}
|
|
s.activeMu.Unlock()
|
|
for _, conn := range conns {
|
|
_ = conn.Close()
|
|
}
|
|
}
|
|
|
|
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) {
|
|
started := time.Now()
|
|
frameStatus := envelope.ParseBadFrame
|
|
defer func() {
|
|
s.recordFrameDuration(frameStatus, time.Since(started))
|
|
}()
|
|
|
|
frameCtx, cancelFrame := context.WithTimeout(context.WithoutCancel(ctx), frameOperationTimeout)
|
|
defer cancelFrame()
|
|
|
|
receivedAtMS := time.Now().UnixMilli()
|
|
var env envelope.FrameEnvelope
|
|
var err error
|
|
if s.protocol.ParseForPlatform != nil {
|
|
platformAccount := ""
|
|
if state != nil {
|
|
platformAccount = state.platformName
|
|
}
|
|
env, err = s.protocol.ParseForPlatform(raw, receivedAtMS, source, platformAccount)
|
|
} else {
|
|
env, err = s.protocol.Parse(raw, receivedAtMS, source)
|
|
}
|
|
if err != nil {
|
|
env = envelope.FrameEnvelope{
|
|
Protocol: s.protocol.Protocol,
|
|
SourceEndpoint: source,
|
|
ReceivedAtMS: receivedAtMS,
|
|
EventTimeMS: receivedAtMS,
|
|
RawHex: strings.ToUpper(hex.EncodeToString(raw)),
|
|
ParseStatus: envelope.ParseBadFrame,
|
|
ParseError: err.Error(),
|
|
}
|
|
env.EventID = env.StableEventID()
|
|
s.recordParseErrorMetric(err)
|
|
} else {
|
|
if s.protocol.Authenticate != nil {
|
|
result := s.protocol.Authenticate.Authenticate(env)
|
|
authentication.Apply(&env, result)
|
|
if result.Applicable {
|
|
s.recordAuthenticationMetric(result)
|
|
}
|
|
}
|
|
authentication.RedactParsedCredentials(&env)
|
|
if envelope.RequiresVehicleIdentity(env) {
|
|
resolveStarted := time.Now()
|
|
resolved, resolveErr := s.resolver.Resolve(frameCtx, env)
|
|
if resolveErr != nil {
|
|
if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" {
|
|
env = resolved
|
|
}
|
|
identityStatus := identityErrorStatus(resolveErr)
|
|
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
|
|
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
|
|
annotateIdentityError(&env, resolveErr)
|
|
env.ParseStatus = envelope.ParsePartial
|
|
s.recordIdentityMetric(identityStatus)
|
|
s.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr))
|
|
} else {
|
|
env = resolved
|
|
annotateIdentityUnresolved(&env)
|
|
identityStatus := identityStatus(env)
|
|
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
|
|
s.recordIdentityMetric(identityStatus)
|
|
if identityStatus != "resolved" {
|
|
s.recordIdentityIssueMetric(identityStatus, env, "no_binding")
|
|
}
|
|
recordGatewayIdentityCacheStatus(s.metrics, s.protocol.Protocol, env)
|
|
}
|
|
} else {
|
|
s.recordIdentitySkipMetric("non_vehicle_frame")
|
|
}
|
|
}
|
|
enrichConnectionPlatform(&env, state)
|
|
frameStatus = env.ParseStatus
|
|
env.EventKind = envelope.EventKindRaw
|
|
s.recordFrameMetric(env.ParseStatus)
|
|
if env.ParseStatus != envelope.ParseBadFrame {
|
|
realtime.EnsureParsedFields(&env)
|
|
}
|
|
|
|
canonicalRaw := canonicalRawEnvelope(env)
|
|
if err := s.sink.PublishRaw(frameCtx, canonicalRaw); err != nil {
|
|
s.recordPublishMetric("raw", "error")
|
|
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
|
return
|
|
}
|
|
s.recordPublishMetric("raw", "ok")
|
|
if env.ParseStatus == envelope.ParseBadFrame {
|
|
return
|
|
}
|
|
s.writeProtocolResponse(conn, raw, env, started)
|
|
if shouldCloseAfterAuthentication(env) && conn != nil {
|
|
_ = conn.Close()
|
|
return
|
|
}
|
|
if s.delegateFields {
|
|
fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env)
|
|
s.recordFieldsMetric(fieldsStatus)
|
|
if ok {
|
|
s.recordFieldsCount(fieldsStatus, fieldCount)
|
|
s.recordPublishMetric("fields", "delegated")
|
|
}
|
|
} else {
|
|
fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env)
|
|
if !ok {
|
|
s.recordFieldsMetric(fieldsStatus)
|
|
} else {
|
|
if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil {
|
|
s.recordFieldsMetric(gatewayFieldsPublishError)
|
|
s.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields))
|
|
s.recordPublishMetric("fields", "error")
|
|
s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
|
} else {
|
|
s.recordFieldsMetric(fieldsStatus)
|
|
s.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields))
|
|
s.recordPublishMetric("fields", "ok")
|
|
}
|
|
}
|
|
}
|
|
if s.publishUnified {
|
|
if err := s.sink.PublishUnified(frameCtx, canonicalRaw); err != nil {
|
|
s.recordPublishMetric("unified", "error")
|
|
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
|
} else {
|
|
s.recordPublishMetric("unified", "ok")
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *TCPServer) writeProtocolResponse(conn net.Conn, raw []byte, env envelope.FrameEnvelope, frameStarted time.Time) {
|
|
if s.protocol.Respond == nil {
|
|
return
|
|
}
|
|
status := "skipped"
|
|
defer func() {
|
|
s.recordResponseDuration(env.MessageID, status, time.Since(frameStarted))
|
|
}()
|
|
response, ok, err := s.protocol.Respond(raw, env)
|
|
if err != nil {
|
|
status = "build_error"
|
|
s.recordResponseMetric(env.MessageID, "build_error")
|
|
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
|
return
|
|
}
|
|
if !ok || len(response) == 0 {
|
|
s.recordResponseMetric(env.MessageID, "skipped")
|
|
return
|
|
}
|
|
if conn == nil {
|
|
status = "write_error"
|
|
s.recordResponseMetric(env.MessageID, "write_error")
|
|
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", "nil connection")
|
|
return
|
|
}
|
|
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
|
|
if _, err := conn.Write(response); err != nil {
|
|
status = "write_error"
|
|
s.recordResponseMetric(env.MessageID, "write_error")
|
|
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
|
|
return
|
|
}
|
|
status = "ok"
|
|
s.recordResponseMetric(env.MessageID, "ok")
|
|
}
|
|
|
|
func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionState) {
|
|
if env == nil || state == nil || env.ParseStatus == envelope.ParseBadFrame {
|
|
return
|
|
}
|
|
if env.Fields == nil {
|
|
env.Fields = map[string]any{}
|
|
}
|
|
if shouldCloseAfterAuthentication(*env) {
|
|
return
|
|
}
|
|
if current := strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])); current != "" && current != "<nil>" {
|
|
state.platformName = current
|
|
}
|
|
if state.platformName == "" {
|
|
return
|
|
}
|
|
if strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])) == "" || fmt.Sprint(env.Fields["platform_account"]) == "<nil>" {
|
|
env.Fields["platform_account"] = state.platformName
|
|
}
|
|
if strings.TrimSpace(env.PlatformName) == "" {
|
|
env.PlatformName = state.platformName
|
|
}
|
|
if strings.TrimSpace(env.SourceCode) == "" {
|
|
env.SourceCode = sourceCodeFromPlatformName(state.platformName)
|
|
}
|
|
if strings.TrimSpace(env.SourceKind) == "" || strings.EqualFold(strings.TrimSpace(env.SourceKind), "UNKNOWN") {
|
|
env.SourceKind = "PLATFORM"
|
|
}
|
|
if env.Parsed == nil {
|
|
env.Parsed = map[string]any{}
|
|
}
|
|
if _, ok := env.Parsed["platform_name"]; !ok {
|
|
env.Parsed["platform_name"] = state.platformName
|
|
}
|
|
}
|
|
|
|
func sourceCodeFromPlatformName(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return ""
|
|
}
|
|
var builder strings.Builder
|
|
for _, r := range value {
|
|
switch {
|
|
case r >= 'a' && r <= 'z':
|
|
builder.WriteRune(r)
|
|
case r >= 'A' && r <= 'Z':
|
|
builder.WriteRune(r)
|
|
case r >= '0' && r <= '9':
|
|
builder.WriteRune(r)
|
|
case r == '_' || r == '-' || r == '.':
|
|
builder.WriteRune(r)
|
|
}
|
|
}
|
|
return builder.String()
|
|
}
|
|
|
|
func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
labels := metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"status": string(status),
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_frames_total", labels)
|
|
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_frame_unix_seconds", labels)
|
|
}
|
|
|
|
func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
elapsedMS := float64(elapsed.Milliseconds())
|
|
labels := metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"status": string(status),
|
|
}
|
|
s.metrics.SetGauge("vehicle_gateway_frame_duration_ms", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"status": string(status),
|
|
}, elapsedMS)
|
|
s.metrics.ObserveHistogram("vehicle_gateway_frame_duration_ms_histogram", labels, gatewayFrameDurationBucketsMS, elapsedMS)
|
|
}
|
|
|
|
func (s *TCPServer) recordPublishMetric(kind string, status string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
labels := metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"kind": kind,
|
|
"status": status,
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_publish_total", labels)
|
|
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_publish_unix_seconds", labels)
|
|
}
|
|
|
|
func (s *TCPServer) recordFieldsMetric(status string) {
|
|
recordGatewayFieldsMetric(s.metrics, s.protocol.Protocol, status)
|
|
}
|
|
|
|
func (s *TCPServer) recordFieldsCount(status string, fieldCount int) {
|
|
recordGatewayFieldsCount(s.metrics, s.protocol.Protocol, status, fieldCount)
|
|
}
|
|
|
|
func (s *TCPServer) recordResponseMetric(messageID string, status string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
messageID = strings.TrimSpace(messageID)
|
|
if messageID == "" {
|
|
messageID = "unknown"
|
|
}
|
|
labels := metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"message_id": messageID,
|
|
"status": status,
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_response_total", labels)
|
|
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_response_unix_seconds", labels)
|
|
}
|
|
|
|
func (s *TCPServer) recordResponseDuration(messageID string, status string, elapsed time.Duration) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
messageID = strings.TrimSpace(messageID)
|
|
if messageID == "" {
|
|
messageID = "unknown"
|
|
}
|
|
elapsedMS := float64(elapsed.Microseconds()) / 1000
|
|
labels := metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"message_id": messageID,
|
|
"status": status,
|
|
}
|
|
s.metrics.ObserveHistogram("vehicle_gateway_response_duration_ms_histogram", labels, gatewayResponseDurationBucketsMS, elapsedMS)
|
|
if status != "ok" {
|
|
return
|
|
}
|
|
protocol := string(s.protocol.Protocol)
|
|
p99, samples := gatewayResponseE2ERecent.Observe(protocol, elapsedMS)
|
|
protocolLabels := metrics.Labels{"protocol": protocol}
|
|
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_p99_ms", protocolLabels, p99)
|
|
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_samples", protocolLabels, float64(samples))
|
|
}
|
|
|
|
func (s *TCPServer) recordAuthenticationMetric(result authentication.Result) {
|
|
if s.metrics == nil || !result.Applicable {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_authentication_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"mode": string(result.Mode),
|
|
"source": result.Source,
|
|
"status": result.Status,
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordParseErrorMetric(err error) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_parse_errors_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"reason": classifyError(err),
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordIdentityMetric(status string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_identity_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"status": status,
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordIdentitySkipMetric(reason string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
messageID := strings.TrimSpace(env.MessageID)
|
|
if messageID == "" {
|
|
messageID = "unknown"
|
|
}
|
|
reason = strings.TrimSpace(reason)
|
|
if reason == "" {
|
|
reason = "unknown"
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"status": status,
|
|
"message_id": messageID,
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordIdentityDuration(status string, elapsed time.Duration) {
|
|
recordGatewayIdentityDuration(s.metrics, s.protocol.Protocol, status, elapsed)
|
|
}
|
|
|
|
func (s *TCPServer) recordConnectionMetric(delta float64) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.AddGauge("vehicle_gateway_active_connections", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
}, delta)
|
|
}
|
|
|
|
func (s *TCPServer) recordConnectionRejection(reason string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_connection_rejections_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func (s *TCPServer) recordConnectionClose(reason string) {
|
|
if s.metrics == nil {
|
|
return
|
|
}
|
|
s.metrics.IncCounter("vehicle_gateway_connection_closes_total", metrics.Labels{
|
|
"protocol": string(s.protocol.Protocol),
|
|
"reason": reason,
|
|
})
|
|
}
|
|
|
|
func shouldCloseAfterAuthentication(env envelope.FrameEnvelope) bool {
|
|
return env.AuthenticationEnforced &&
|
|
strings.TrimSpace(env.AuthenticationStatus) != "" &&
|
|
env.AuthenticationStatus != authentication.StatusAccepted
|
|
}
|
|
|
|
func identityStatus(env envelope.FrameEnvelope) string {
|
|
if strings.TrimSpace(env.VIN) != "" {
|
|
return "resolved"
|
|
}
|
|
if env.ParseStatus == envelope.ParsePartial {
|
|
return "error"
|
|
}
|
|
return "unresolved"
|
|
}
|
|
|
|
func identityIssueReason(status string, err error) string {
|
|
switch status {
|
|
case "timeout":
|
|
return "timeout"
|
|
case "error":
|
|
if err == nil {
|
|
return "resolver_error"
|
|
}
|
|
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
|
switch {
|
|
case strings.Contains(text, "connection"), strings.Contains(text, "tcp"), strings.Contains(text, "network"):
|
|
return "identity_store_connection"
|
|
case strings.Contains(text, "timeout"), strings.Contains(text, "deadline"):
|
|
return "timeout"
|
|
default:
|
|
return "resolver_error"
|
|
}
|
|
default:
|
|
return "no_binding"
|
|
}
|
|
}
|
|
|
|
func annotateIdentityUnresolved(env *envelope.FrameEnvelope) {
|
|
if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" {
|
|
return
|
|
}
|
|
if env.Parsed == nil {
|
|
env.Parsed = map[string]any{}
|
|
}
|
|
if _, exists := env.Parsed["identity"]; exists {
|
|
return
|
|
}
|
|
env.Parsed["identity"] = map[string]any{
|
|
"resolved": false,
|
|
"reason": "no_binding",
|
|
}
|
|
}
|
|
|
|
func classifyError(err error) string {
|
|
text := strings.ToLower(strings.TrimSpace(fmt.Sprint(err)))
|
|
switch {
|
|
case text == "":
|
|
return "unknown"
|
|
case strings.Contains(text, "bcc") || strings.Contains(text, "checksum"):
|
|
return "checksum"
|
|
case strings.Contains(text, "short") || strings.Contains(text, "truncated") || strings.Contains(text, "length"):
|
|
return "length"
|
|
case strings.Contains(text, "json"):
|
|
return "json"
|
|
case strings.Contains(text, "start"):
|
|
return "start_symbol"
|
|
default:
|
|
return "parse"
|
|
}
|
|
}
|
|
|
|
func (p TCPProtocol) String() string {
|
|
return fmt.Sprintf("%s@%s", p.Protocol, p.Addr)
|
|
}
|