shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

368
go/vehicle-gateway/internal/gateway/tcp_server.go
View File

@@ -10,8 +10,10 @@ import (
"net"
"strings"
"sync"
"syscall"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/authentication"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/identity"
@@ -26,11 +28,12 @@ type FrameParser func(raw []byte, receivedAtMS int64, sourceEndpoint string) (en
type FrameResponder func(raw []byte, env envelope.FrameEnvelope) (response []byte, ok bool, err error)
type TCPProtocol struct {
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Respond FrameResponder
Protocol envelope.Protocol
Addr string
Extract FrameExtractor
Parse FrameParser
Authenticate authentication.Authenticator
Respond FrameResponder
}
type connectionState struct {
@@ -47,6 +50,9 @@ type TCPServer struct {
idleTimeout time.Duration
maxConnections int
publishUnified bool
delegateFields bool
activeMu sync.Mutex
activeConns map[net.Conn]struct{}
}
type TCPServerConfig struct {
@@ -59,11 +65,14 @@ type TCPServerConfig struct {
IdleTimeout time.Duration
MaxConnections int
PublishUnified bool
DelegateFields bool
}
const frameOperationTimeout = 30 * time.Second
var gatewayFrameDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
var gatewayResponseDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
var gatewayResponseE2ERecent = metrics.NewRecentLatencyByKey(512)
func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
if cfg.Protocol.Protocol == "" {
@@ -106,6 +115,8 @@ func NewTCPServer(cfg TCPServerConfig) (*TCPServer, error) {
idleTimeout: cfg.IdleTimeout,
maxConnections: cfg.MaxConnections,
publishUnified: cfg.PublishUnified,
delegateFields: cfg.DelegateFields,
activeConns: map[net.Conn]struct{}{},
}, nil
}
@@ -120,6 +131,7 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
go func() {
<-ctx.Done()
_ = listener.Close()
s.closeActiveConnections()
}()
s.logger.Info("tcp listener started", "protocol", s.protocol.Protocol, "addr", listener.Addr().String())
@@ -155,12 +167,15 @@ func (s *TCPServer) ListenAndServe(ctx context.Context) error {
func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
defer conn.Close()
s.trackConnection(conn)
defer s.untrackConnection(conn)
source := conn.RemoteAddr().String()
log := s.logger.With("protocol", s.protocol.Protocol, "remote", source)
s.recordConnectionMetric(1)
defer s.recordConnectionMetric(-1)
log.Info("tcp connection opened")
defer log.Info("tcp connection closed")
log.Debug("tcp connection opened")
defer log.Debug("tcp connection closed")
readBuffer := make([]byte, s.readBufferSize)
var pending []byte
@@ -182,16 +197,21 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
}
if err != nil {
if errors.Is(err, io.EOF) {
s.recordConnectionClose("eof")
if ctx.Err() != nil {
s.recordConnectionClose("context_cancelled")
return
}
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
log.Warn("tcp connection idle timeout")
log.Debug("tcp connection idle timeout")
s.recordConnectionClose("read_timeout")
return
}
if isRoutineTCPReadClose(err) {
log.Debug("tcp connection closed by peer", "error", err)
s.recordConnectionClose("remote_closed")
return
}
log.Warn("tcp read failed", "error", err)
s.recordConnectionClose("read_error")
return
@@ -203,6 +223,55 @@ func (s *TCPServer) handleConnection(ctx context.Context, conn net.Conn) {
}
}
func isRoutineTCPReadClose(err error) bool {
if err == nil {
return false
}
if errors.Is(err, io.EOF) ||
errors.Is(err, net.ErrClosed) ||
errors.Is(err, syscall.ECONNRESET) ||
errors.Is(err, syscall.EPIPE) {
return true
}
text := strings.ToLower(err.Error())
return strings.Contains(text, "connection reset by peer") ||
strings.Contains(text, "broken pipe") ||
strings.Contains(text, "use of closed network connection")
}
func (s *TCPServer) trackConnection(conn net.Conn) {
if conn == nil {
return
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
if s.activeConns == nil {
s.activeConns = map[net.Conn]struct{}{}
}
s.activeConns[conn] = struct{}{}
}
func (s *TCPServer) untrackConnection(conn net.Conn) {
if conn == nil {
return
}
s.activeMu.Lock()
defer s.activeMu.Unlock()
delete(s.activeConns, conn)
}
func (s *TCPServer) closeActiveConnections() {
s.activeMu.Lock()
conns := make([]net.Conn, 0, len(s.activeConns))
for conn := range s.activeConns {
conns = append(conns, conn)
}
s.activeMu.Unlock()
for _, conn := range conns {
_ = conn.Close()
}
}
func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte, source string, state *connectionState) {
started := time.Now()
frameStatus := envelope.ParseBadFrame
@@ -228,29 +297,53 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
env.EventID = env.StableEventID()
s.recordParseErrorMetric(err)
} else {
resolved, resolveErr := s.resolver.Resolve(frameCtx, env)
if resolveErr != nil {
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
if env.Parsed == nil {
env.Parsed = map[string]any{}
if s.protocol.Authenticate != nil {
result := s.protocol.Authenticate.Authenticate(env)
authentication.Apply(&env, result)
if result.Applicable {
s.recordAuthenticationMetric(result)
}
env.Parsed["identity"] = map[string]any{"resolved": false, "error": resolveErr.Error()}
env.ParseStatus = envelope.ParsePartial
s.recordIdentityMetric("error")
} else {
env = resolved
annotateIdentityUnresolved(&env)
s.recordIdentityMetric(identityStatus(env))
}
enrichConnectionPlatform(&env, state)
authentication.RedactParsedCredentials(&env)
if envelope.RequiresVehicleIdentity(env) {
resolveStarted := time.Now()
resolved, resolveErr := s.resolver.Resolve(frameCtx, env)
if resolveErr != nil {
if resolved.Protocol != "" || strings.TrimSpace(resolved.VIN) != "" || strings.TrimSpace(resolved.Phone) != "" {
env = resolved
}
identityStatus := identityErrorStatus(resolveErr)
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
s.logger.Warn("identity resolve failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", resolveErr)
annotateIdentityError(&env, resolveErr)
env.ParseStatus = envelope.ParsePartial
s.recordIdentityMetric(identityStatus)
s.recordIdentityIssueMetric(identityStatus, env, identityIssueReason(identityStatus, resolveErr))
} else {
env = resolved
annotateIdentityUnresolved(&env)
identityStatus := identityStatus(env)
s.recordIdentityDuration(identityStatus, time.Since(resolveStarted))
s.recordIdentityMetric(identityStatus)
if identityStatus != "resolved" {
s.recordIdentityIssueMetric(identityStatus, env, "no_binding")
}
recordGatewayIdentityCacheStatus(s.metrics, s.protocol.Protocol, env)
}
} else {
s.recordIdentitySkipMetric("non_vehicle_frame")
}
}
enrichConnectionPlatform(&env, state)
frameStatus = env.ParseStatus
env.EventKind = envelope.EventKindRaw
s.recordFrameMetric(env.ParseStatus)
if env.ParseStatus != envelope.ParseBadFrame {
realtime.EnsureParsedFields(&env)
}
if err := s.sink.PublishRaw(frameCtx, env); err != nil {
canonicalRaw := canonicalRawEnvelope(env)
if err := s.sink.PublishRaw(frameCtx, canonicalRaw); err != nil {
s.recordPublishMetric("raw", "error")
s.logger.Error("publish raw failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
@@ -259,37 +352,79 @@ func (s *TCPServer) handleFrame(ctx context.Context, conn net.Conn, raw []byte,
if env.ParseStatus == envelope.ParseBadFrame {
return
}
if fieldsEnv, ok := realtime.BuildFieldsEnvelope(env); ok {
if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil {
s.recordPublishMetric("fields", "error")
s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
s.writeProtocolResponse(conn, raw, env, started)
if shouldCloseAfterAuthentication(env) && conn != nil {
_ = conn.Close()
return
}
if s.delegateFields {
fieldCount, fieldsStatus, ok := gatewayDelegatedFields(env)
s.recordFieldsMetric(fieldsStatus)
if ok {
s.recordFieldsCount(fieldsStatus, fieldCount)
s.recordPublishMetric("fields", "delegated")
}
} else {
fieldsEnv, fieldsStatus, ok := gatewayFieldsEnvelope(env)
if !ok {
s.recordFieldsMetric(fieldsStatus)
} else {
if err := s.sink.PublishFields(frameCtx, fieldsEnv); err != nil {
s.recordFieldsMetric(gatewayFieldsPublishError)
s.recordFieldsCount(gatewayFieldsPublishError, len(fieldsEnv.Fields))
s.recordPublishMetric("fields", "error")
s.logger.Error("publish fields failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
} else {
s.recordFieldsMetric(fieldsStatus)
s.recordFieldsCount(fieldsStatus, len(fieldsEnv.Fields))
s.recordPublishMetric("fields", "ok")
}
}
s.recordPublishMetric("fields", "ok")
}
if s.publishUnified {
if err := s.sink.PublishUnified(frameCtx, env); err != nil {
if err := s.sink.PublishUnified(frameCtx, canonicalRaw); err != nil {
s.recordPublishMetric("unified", "error")
s.logger.Error("publish unified failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
} else {
s.recordPublishMetric("unified", "ok")
}
s.recordPublishMetric("unified", "ok")
}
}
func (s *TCPServer) writeProtocolResponse(conn net.Conn, raw []byte, env envelope.FrameEnvelope, frameStarted time.Time) {
if s.protocol.Respond == nil {
return
}
status := "skipped"
defer func() {
s.recordResponseDuration(env.MessageID, status, time.Since(frameStarted))
}()
response, ok, err := s.protocol.Respond(raw, env)
if err != nil {
status = "build_error"
s.recordResponseMetric(env.MessageID, "build_error")
s.logger.Warn("build protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
if !ok || len(response) == 0 {
s.recordResponseMetric(env.MessageID, "skipped")
return
}
if conn == nil {
status = "write_error"
s.recordResponseMetric(env.MessageID, "write_error")
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", "nil connection")
return
}
_ = conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(response); err != nil {
status = "write_error"
s.recordResponseMetric(env.MessageID, "write_error")
s.logger.Warn("write protocol response failed", "protocol", s.protocol.Protocol, "event_id", env.StableEventID(), "error", err)
return
}
status = "ok"
s.recordResponseMetric(env.MessageID, "ok")
}
func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionState) {
@@ -299,6 +434,9 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat
if env.Fields == nil {
env.Fields = map[string]any{}
}
if shouldCloseAfterAuthentication(*env) {
return
}
if current := strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])); current != "" && current != "<nil>" {
state.platformName = current
}
@@ -308,6 +446,15 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat
if strings.TrimSpace(fmt.Sprint(env.Fields["platform_account"])) == "" || fmt.Sprint(env.Fields["platform_account"]) == "<nil>" {
env.Fields["platform_account"] = state.platformName
}
if strings.TrimSpace(env.PlatformName) == "" {
env.PlatformName = state.platformName
}
if strings.TrimSpace(env.SourceCode) == "" {
env.SourceCode = sourceCodeFromPlatformName(state.platformName)
}
if strings.TrimSpace(env.SourceKind) == "" || strings.EqualFold(strings.TrimSpace(env.SourceKind), "UNKNOWN") {
env.SourceKind = "PLATFORM"
}
if env.Parsed == nil {
env.Parsed = map[string]any{}
}
@@ -316,14 +463,37 @@ func enrichConnectionPlatform(env *envelope.FrameEnvelope, state *connectionStat
}
}
func sourceCodeFromPlatformName(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
var builder strings.Builder
for _, r := range value {
switch {
case r >= 'a' && r <= 'z':
builder.WriteRune(r)
case r >= 'A' && r <= 'Z':
builder.WriteRune(r)
case r >= '0' && r <= '9':
builder.WriteRune(r)
case r == '_' || r == '-' || r == '.':
builder.WriteRune(r)
}
}
return builder.String()
}
func (s *TCPServer) recordFrameMetric(status envelope.ParseStatus) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_frames_total", metrics.Labels{
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": string(status),
})
}
s.metrics.IncCounter("vehicle_gateway_frames_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_frame_unix_seconds", labels)
}
func (s *TCPServer) recordFrameDuration(status envelope.ParseStatus, elapsed time.Duration) {
@@ -346,10 +516,74 @@ func (s *TCPServer) recordPublishMetric(kind string, status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_publish_total", metrics.Labels{
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"kind": kind,
"status": status,
}
s.metrics.IncCounter("vehicle_gateway_publish_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_publish_unix_seconds", labels)
}
func (s *TCPServer) recordFieldsMetric(status string) {
recordGatewayFieldsMetric(s.metrics, s.protocol.Protocol, status)
}
func (s *TCPServer) recordFieldsCount(status string, fieldCount int) {
recordGatewayFieldsCount(s.metrics, s.protocol.Protocol, status, fieldCount)
}
func (s *TCPServer) recordResponseMetric(messageID string, status string) {
if s.metrics == nil {
return
}
messageID = strings.TrimSpace(messageID)
if messageID == "" {
messageID = "unknown"
}
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"message_id": messageID,
"status": status,
}
s.metrics.IncCounter("vehicle_gateway_response_total", labels)
metrics.RecordLastActivity(s.metrics, "vehicle_gateway_last_response_unix_seconds", labels)
}
func (s *TCPServer) recordResponseDuration(messageID string, status string, elapsed time.Duration) {
if s.metrics == nil {
return
}
messageID = strings.TrimSpace(messageID)
if messageID == "" {
messageID = "unknown"
}
elapsedMS := float64(elapsed.Microseconds()) / 1000
labels := metrics.Labels{
"protocol": string(s.protocol.Protocol),
"message_id": messageID,
"status": status,
}
s.metrics.ObserveHistogram("vehicle_gateway_response_duration_ms_histogram", labels, gatewayResponseDurationBucketsMS, elapsedMS)
if status != "ok" {
return
}
protocol := string(s.protocol.Protocol)
p99, samples := gatewayResponseE2ERecent.Observe(protocol, elapsedMS)
protocolLabels := metrics.Labels{"protocol": protocol}
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_p99_ms", protocolLabels, p99)
s.metrics.SetGauge("vehicle_gateway_response_e2e_recent_samples", protocolLabels, float64(samples))
}
func (s *TCPServer) recordAuthenticationMetric(result authentication.Result) {
if s.metrics == nil || !result.Applicable {
return
}
s.metrics.IncCounter("vehicle_gateway_authentication_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"mode": string(result.Mode),
"source": result.Source,
"status": result.Status,
})
}
@@ -373,6 +607,40 @@ func (s *TCPServer) recordIdentityMetric(status string) {
})
}
func (s *TCPServer) recordIdentitySkipMetric(reason string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_gateway_identity_skips_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"reason": reason,
})
}
func (s *TCPServer) recordIdentityIssueMetric(status string, env envelope.FrameEnvelope, reason string) {
if s.metrics == nil {
return
}
messageID := strings.TrimSpace(env.MessageID)
if messageID == "" {
messageID = "unknown"
}
reason = strings.TrimSpace(reason)
if reason == "" {
reason = "unknown"
}
s.metrics.IncCounter("vehicle_gateway_identity_issues_total", metrics.Labels{
"protocol": string(s.protocol.Protocol),
"status": status,
"message_id": messageID,
"reason": reason,
})
}
func (s *TCPServer) recordIdentityDuration(status string, elapsed time.Duration) {
recordGatewayIdentityDuration(s.metrics, s.protocol.Protocol, status, elapsed)
}
func (s *TCPServer) recordConnectionMetric(delta float64) {
if s.metrics == nil {
return
@@ -402,6 +670,12 @@ func (s *TCPServer) recordConnectionClose(reason string) {
})
}
func shouldCloseAfterAuthentication(env envelope.FrameEnvelope) bool {
return env.AuthenticationEnforced &&
strings.TrimSpace(env.AuthenticationStatus) != "" &&
env.AuthenticationStatus != authentication.StatusAccepted
}
func identityStatus(env envelope.FrameEnvelope) string {
if strings.TrimSpace(env.VIN) != "" {
return "resolved"
@@ -412,6 +686,28 @@ func identityStatus(env envelope.FrameEnvelope) string {
return "unresolved"
}
func identityIssueReason(status string, err error) string {
switch status {
case "timeout":
return "timeout"
case "error":
if err == nil {
return "resolver_error"
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
switch {
case strings.Contains(text, "connection"), strings.Contains(text, "tcp"), strings.Contains(text, "network"):
return "identity_store_connection"
case strings.Contains(text, "timeout"), strings.Contains(text, "deadline"):
return "timeout"
default:
return "resolver_error"
}
default:
return "no_binding"
}
}
func annotateIdentityUnresolved(env *envelope.FrameEnvelope) {
if env == nil || env.ParseStatus == envelope.ParseBadFrame || strings.TrimSpace(env.VIN) != "" {
return