shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
go/vehicle-gateway/internal/stats/source.go
View File

@@ -12,6 +12,9 @@ type SourceIdentity struct {
Protocol envelope.Protocol
SourceIP string
SourceEndpoint string
SourceCode string
PlatformName string
SourceKind string
}
func NewSourceIdentity(protocol envelope.Protocol, endpoint string) (SourceIdentity, bool) {
@@ -26,15 +29,29 @@ func NewSourceIdentity(protocol envelope.Protocol, endpoint string) (SourceIdent
}, true
}
func NewSourceIdentityFromEnvelope(env envelope.FrameEnvelope) (SourceIdentity, bool) {
identity, ok := NewSourceIdentity(env.Protocol, env.SourceEndpoint)
if !ok {
return SourceIdentity{}, false
}
identity.SourceCode = strings.TrimSpace(env.SourceCode)
identity.PlatformName = strings.TrimSpace(env.PlatformName)
identity.SourceKind = normalizeSourceKindForRead(env.SourceKind)
return identity, true
}
func ShouldManageDataSource(identity SourceIdentity) bool {
if strings.TrimSpace(identity.SourceIP) == "" {
return false
}
if strings.TrimSpace(identity.SourceCode) != "" || strings.TrimSpace(identity.PlatformName) != "" {
return true
}
return normalizeSourceKindForRead(identity.SourceKind) == "PLATFORM"
}
func NormalizeSourceIP(endpoint string) string {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
return ""
}
if host, _, ok := strings.Cut(endpoint, ":"); ok {
return strings.TrimSpace(host)
}
return endpoint
return envelope.NormalizeSourceEndpointKey(endpoint)
}
func UpsertDataSource(ctx context.Context, exec Execer, identity SourceIdentity, now time.Time) error {
@@ -51,18 +68,69 @@ func UpsertDataSource(ctx context.Context, exec Execer, identity SourceIdentity,
string(identity.Protocol),
identity.SourceIP,
identity.SourceEndpoint,
nullableTrimmedString(identity.PlatformName),
nullableTrimmedString(identity.SourceCode),
sourceKindForDataSourceWrite(identity),
now,
now,
)
return err
}
func sourceKindForDataSourceWrite(identity SourceIdentity) string {
kind := normalizeSourceKindForWrite(identity.SourceKind)
if kind != "UNKNOWN" {
return kind
}
if strings.TrimSpace(identity.SourceCode) != "" || strings.TrimSpace(identity.PlatformName) != "" {
return "PLATFORM"
}
return "UNKNOWN"
}
const upsertDataSourceSQL = `
INSERT INTO vehicle_data_source
(protocol, source_ip, latest_source_endpoint, first_seen_at, latest_seen_at)
VALUES (?, ?, ?, ?, ?)
(protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, first_seen_at, latest_seen_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
latest_source_endpoint = VALUES(latest_source_endpoint),
latest_seen_at = VALUES(latest_seen_at),
platform_name = CASE
WHEN vehicle_data_source.platform_name IS NULL OR TRIM(vehicle_data_source.platform_name) = ''
THEN VALUES(platform_name)
ELSE vehicle_data_source.platform_name
END,
source_code = CASE
WHEN vehicle_data_source.source_code IS NULL OR TRIM(vehicle_data_source.source_code) = ''
THEN VALUES(source_code)
ELSE vehicle_data_source.source_code
END,
source_kind = CASE
WHEN vehicle_data_source.source_kind IS NULL OR TRIM(vehicle_data_source.source_kind) = '' OR vehicle_data_source.source_kind = 'UNKNOWN'
THEN VALUES(source_kind)
ELSE vehicle_data_source.source_kind
END,
enabled = CASE
WHEN vehicle_data_source.enabled = 0
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
AND (
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
OR VALUES(source_kind) <> 'UNKNOWN'
)
THEN 1
ELSE vehicle_data_source.enabled
END,
remark = CASE
WHEN vehicle_data_source.enabled = 0
AND (vehicle_data_source.remark LIKE 'auto-retired:%' OR vehicle_data_source.remark = 'auto-reenabled: source evidence restored')
AND (
(VALUES(platform_name) IS NOT NULL AND TRIM(VALUES(platform_name)) <> '')
OR (VALUES(source_code) IS NOT NULL AND TRIM(VALUES(source_code)) <> '')
OR VALUES(source_kind) <> 'UNKNOWN'
)
THEN 'auto-reenabled: source evidence restored'
ELSE vehicle_data_source.remark
END,
latest_seen_at = GREATEST(COALESCE(latest_seen_at, VALUES(latest_seen_at)), VALUES(latest_seen_at)),
updated_at = CURRENT_TIMESTAMP
`