shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
go/vehicle-gateway/internal/stats/source_test.go
View File

@@ -11,10 +11,12 @@ import (
func TestNormalizeSourceIPDropsPort(t *testing.T) {
tests := map[string]string{
"115.231.168.135:20215": "115.231.168.135",
"115.231.168.135": "115.231.168.135",
" 115.159.85.149:28316 ": "115.159.85.149",
"": "",
"115.231.168.135:20215": "115.231.168.135",
"115.231.168.135": "115.231.168.135",
" 115.159.85.149:28316 ": "115.159.85.149",
"mqtt://yutong/ytforward/shln/3": "mqtt",
"MQTT://YUTONG/topic": "mqtt",
"": "",
}
for input, want := range tests {
if got := NormalizeSourceIP(input); got != want {
@@ -43,6 +45,60 @@ func TestNewSourceIdentityRequiresSourceIP(t *testing.T) {
}
}
func TestShouldManageDataSourceRequiresPlatformEvidence(t *testing.T) {
if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135"}) {
t.Fatal("unclassified source should not be managed")
}
if !ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "g7s"}) {
t.Fatal("source_code should make source manageable")
}
if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceKind: "DIRECT"}) {
t.Fatal("direct source without platform evidence should not be auto-managed by source IP")
}
if !ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceKind: "PLATFORM"}) {
t.Fatal("explicit platform source_kind should make source manageable")
}
if ShouldManageDataSource(SourceIdentity{Protocol: envelope.ProtocolJT808, SourceCode: "g7s"}) {
t.Fatal("empty source ip should not be managed")
}
}
func TestSourceKindForDataSourceWriteInfersPlatformEvidence(t *testing.T) {
tests := []struct {
name string
identity SourceIdentity
want string
}{
{
name: "empty evidence",
identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135"},
want: "UNKNOWN",
},
{
name: "source code",
identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "g7s"},
want: "PLATFORM",
},
{
name: "platform name",
identity: SourceIdentity{Protocol: envelope.ProtocolGB32960, SourceIP: "8.134.95.166", PlatformName: "Hyundai"},
want: "PLATFORM",
},
{
name: "explicit direct wins",
identity: SourceIdentity{Protocol: envelope.ProtocolJT808, SourceIP: "115.231.168.135", SourceCode: "direct-import", SourceKind: "DIRECT"},
want: "DIRECT",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := sourceKindForDataSourceWrite(tt.identity); got != tt.want {
t.Fatalf("sourceKindForDataSourceWrite() = %q, want %q", got, tt.want)
}
})
}
}
func TestUpsertDataSourcePreservesManualFields(t *testing.T) {
exec := &recordingExec{}
identity := SourceIdentity{
@@ -60,7 +116,7 @@ func TestUpsertDataSourcePreservesManualFields(t *testing.T) {
for _, want := range []string{
"INSERT INTO vehicle_data_source",
"latest_source_endpoint = VALUES(latest_source_endpoint)",
"latest_seen_at = VALUES(latest_seen_at)",
"latest_seen_at = GREATEST",
} {
if !strings.Contains(sql, want) {
t.Fatalf("source upsert missing %q: %s", want, sql)
@@ -77,3 +133,103 @@ func TestUpsertDataSourcePreservesManualFields(t *testing.T) {
}
}
}
func TestUpsertDataSourceWritesPlatformKindWhenEvidenceExists(t *testing.T) {
exec := &recordingExec{}
identity := SourceIdentity{
Protocol: envelope.ProtocolGB32960,
SourceIP: "8.134.95.166",
SourceEndpoint: "8.134.95.166:32960",
SourceCode: "Hyundai",
PlatformName: "现代 HTWO",
}
if err := UpsertDataSource(context.Background(), exec, identity, time.Date(2026, 7, 12, 18, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("UpsertDataSource() error = %v", err)
}
if len(exec.calls) != 1 {
t.Fatalf("exec calls = %d", len(exec.calls))
}
if got, want := exec.calls[0].args[5], "PLATFORM"; got != want {
t.Fatalf("source_kind arg = %#v, want %q", got, want)
}
}
func TestUpsertDataSourceCanReenableAutoRetiredSourceWithEvidence(t *testing.T) {
exec := &recordingExec{}
identity := SourceIdentity{
Protocol: envelope.ProtocolJT808,
SourceIP: "115.231.168.135",
SourceEndpoint: "115.231.168.135:20215",
SourceCode: "g7s",
PlatformName: "G7s",
}
if err := UpsertDataSource(context.Background(), exec, identity, time.Date(2026, 7, 12, 18, 0, 0, 0, time.UTC)); err != nil {
t.Fatalf("UpsertDataSource() error = %v", err)
}
sql := exec.calls[0].query
for _, want := range []string{
"vehicle_data_source.enabled = 0",
"vehicle_data_source.remark LIKE 'auto-retired:%'",
"vehicle_data_source.remark = 'auto-reenabled: source evidence restored'",
"THEN 'auto-reenabled: source evidence restored'",
"THEN 1",
} {
if !strings.Contains(sql, want) {
t.Fatalf("source upsert should re-enable auto-retired source with evidence; missing %q:\n%s", want, sql)
}
}
if strings.Contains(sql, "enabled = VALUES(enabled)") {
t.Fatalf("source upsert should not blindly copy enabled from values:\n%s", sql)
}
if strings.Index(sql, "enabled = CASE") < 0 || strings.Index(sql, "remark = CASE") < 0 || strings.Index(sql, "enabled = CASE") > strings.Index(sql, "remark = CASE") {
t.Fatalf("source upsert must restore enabled before updating remark because MySQL evaluates assignments in order:\n%s", sql)
}
}
func TestDataSourceSchemaIncludesStableSourceCode(t *testing.T) {
for _, want := range []string{
"source_code VARCHAR(64) NULL",
"source_kind VARCHAR(32) NOT NULL DEFAULT 'UNKNOWN'",
"KEY idx_protocol_source_code (protocol, source_code)",
"KEY idx_protocol_source_kind_seen (protocol, source_kind, latest_seen_at)",
} {
if !strings.Contains(DataSourceTableSQL, want) {
t.Fatalf("data source schema missing %q:\n%s", want, DataSourceTableSQL)
}
}
for _, want := range []string{
"ALTER TABLE vehicle_data_source ADD COLUMN source_code",
"ALTER TABLE vehicle_data_source ADD COLUMN source_kind",
"ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_code",
"ALTER TABLE vehicle_data_source ADD KEY idx_protocol_source_kind_seen",
} {
if !containsStatement(DailyMileageAlterSQL, want) {
t.Fatalf("data source alter SQL missing %q: %#v", want, DailyMileageAlterSQL)
}
}
}
func TestDailyMileageSourceSchemaIncludesQueryIndexes(t *testing.T) {
for _, want := range []string{
"KEY idx_protocol_date_selected (protocol, stat_date, is_selected, vin)",
"KEY idx_protocol_date_quality (protocol, stat_date, quality_status, vin)",
"KEY idx_protocol_date_quality_reason (protocol, stat_date, quality_status, quality_reason, vin)",
"KEY idx_source_ip_date (protocol, source_ip, stat_date)",
} {
if !strings.Contains(DailyMileageSourceTableSQL, want) {
t.Fatalf("daily mileage source schema missing %q:\n%s", want, DailyMileageSourceTableSQL)
}
if !containsStatement(DailyMileageAlterSQL, "ALTER TABLE vehicle_daily_mileage_source ADD "+want) {
t.Fatalf("daily mileage source alter SQL missing %q: %#v", want, DailyMileageAlterSQL)
}
}
}
func containsStatement(statements []string, fragment string) bool {
for _, statement := range statements {
if strings.Contains(statement, fragment) {
return true
}
}
return false
}