shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
go/vehicle-gateway/internal/realtime/repository_test.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
@@ -133,6 +134,50 @@ func TestRepositoryStoresFullParsedOnlyInRealtimeRawAndMergesGB32960Units(t *tes
}
}
func TestRepositoryNormalizesFarFutureEventTime(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli()
futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC).UnixMilli()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: futureEvent,
ReceivedAtMS: received,
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1}},
ParsedFields: map[string]any{
"jt808.location.longitude": 121.1,
"jt808.location.latitude": 30.1,
"jt808.location.total_mileage_km": 10241.2,
},
Fields: map[string]any{
envelope.FieldLongitude: 121.1,
envelope.FieldLatitude: 30.1,
envelope.FieldTotalMileageKM: 10241.2,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil {
t.Fatalf("GetProtocol() error = %v", err)
}
if protocol.EventTimeMS != received {
t.Fatalf("protocol event time = %d, want received %d", protocol.EventTimeMS, received)
}
meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("HGetAll meta error = %v", err)
}
if meta["event_time_ms"] != strconv.FormatInt(received, 10) {
t.Fatalf("rt-kv event_time_ms = %q, want received %d", meta["event_time_ms"], received)
}
}
func TestRepositoryMergesNestedGB32960MotorSlicesBySerialNo(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
@@ -337,6 +382,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) {
}},
},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
"gb32960.gd_fc_stack.type": "0x30",
"gb32960.gd_fc_stack.stack_count": 1,
"gb32960.gd_fc_stack.stack_water_outlet_temp_c": 63,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
@@ -358,6 +410,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) {
if typeKV["gb32960.vehicle.soc_percent"] != "number" || typeKV["gb32960.gd_fc_stack.type"] != "string" {
t.Fatalf("types kv = %#v", typeKV)
}
timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result()
if err != nil {
t.Fatalf("times kv HGetAll error = %v", err)
}
if timeKV["gb32960.vehicle.soc_percent"] != "1000" || timeKV["gb32960.gd_fc_stack.stack_water_outlet_temp_c"] != "1000" {
t.Fatalf("times kv = %#v", timeKV)
}
metaKV, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:meta").Result()
if err != nil {
t.Fatalf("meta kv HGetAll error = %v", err)
@@ -388,6 +447,10 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}},
},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
}
@@ -406,9 +469,19 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
if typeKV["gb32960.vehicle.soc_percent"] != "number" {
t.Fatalf("types kv = %#v", typeKV)
}
timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result()
if err != nil {
t.Fatalf("times kv HGetAll error = %v", err)
}
if timeKV["gb32960.vehicle.soc_percent"] != "1000" {
t.Fatalf("times kv = %#v", timeKV)
}
if ttl := repo.client.TTL(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val(); ttl != -1 {
t.Fatalf("kv ttl should not expire, got %v", ttl)
}
if ttl := repo.client.TTL(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Val(); ttl != -1 {
t.Fatalf("kv times ttl should not expire, got %v", ttl)
}
if ttl := repo.client.TTL(ctx, "vehicle:online:GB32960:VIN001").Val(); ttl <= 0 || ttl > time.Minute {
t.Fatalf("online ttl = %v, want within 1 minute", ttl)
}
@@ -430,12 +503,138 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
}
}
func TestRepositoryFastUpdateDoesNotLetOlderFramesOverwriteNewerFields(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
firstResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: 2000,
ReceivedAtMS: 2100,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 50,
},
})
if err != nil {
t.Fatalf("newer FastUpdate() error = %v", err)
}
if firstResult.FieldsSeen != 1 || firstResult.FieldsWritten != 1 || firstResult.FieldsSkippedStale != 0 {
t.Fatalf("newer result = %#v, want 1/1/0", firstResult)
}
if firstResult.EnvelopesSeen != 1 || firstResult.EnvelopesUpdated != 1 {
t.Fatalf("newer envelope result = %#v, want seen=1 updated=1", firstResult)
}
secondResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 2200,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 10,
"jt808.location.latitude": 30.5,
},
})
if err != nil {
t.Fatalf("older FastUpdate() error = %v", err)
}
if secondResult.FieldsSeen != 2 || secondResult.FieldsWritten != 1 || secondResult.FieldsSkippedStale != 1 {
t.Fatalf("older result = %#v, want 2/1/1", secondResult)
}
if secondResult.EnvelopesSeen != 1 || secondResult.EnvelopesUpdated != 1 {
t.Fatalf("older envelope result = %#v, want seen=1 updated=1", secondResult)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("values HGetAll error = %v", err)
}
if values["jt808.location.speed_kmh"] != "50" {
t.Fatalf("older frame overwrote speed: %#v", values)
}
if values["jt808.location.latitude"] != "30.5" {
t.Fatalf("older frame should still add previously unseen latitude: %#v", values)
}
times, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("times HGetAll error = %v", err)
}
if times["jt808.location.speed_kmh"] != "2000" || times["jt808.location.latitude"] != "1000" {
t.Fatalf("field times = %#v", times)
}
meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("meta HGetAll error = %v", err)
}
if meta["event_time_ms"] != "2000" {
t.Fatalf("meta should keep latest frame time: %#v", meta)
}
}
func TestRepositoryFastUpdateDoesNotLetOlderReceivedFrameOverwriteOnlineState(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
SourceEndpoint: "newer.example:808",
EventTimeMS: 2000,
ReceivedAtMS: 5000,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 50,
},
}); err != nil {
t.Fatalf("newer FastUpdate() error = %v", err)
}
if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
SourceEndpoint: "older.example:808",
EventTimeMS: 1000,
ReceivedAtMS: 4000,
ParsedFields: map[string]any{
"jt808.location.latitude": 30.5,
},
}); err != nil {
t.Fatalf("older FastUpdate() error = %v", err)
}
state, err := repo.client.HGetAll(ctx, "vehicle:online-state:JT808:VIN001").Result()
if err != nil {
t.Fatalf("online state HGetAll error = %v", err)
}
if state["last_seen_ms"] != "5000" || state["offline_after_ms"] != "65000" || state["source_endpoint"] != "newer.example:808" {
t.Fatalf("older received frame should not overwrite online state: %#v", state)
}
score, err := repo.client.ZScore(ctx, "vehicle:last_seen", "JT808:VIN001").Result()
if err != nil {
t.Fatalf("last_seen ZScore error = %v", err)
}
if score != 5000 {
t.Fatalf("last_seen score = %v, want 5000", score)
}
status, err := repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if status.LastSeenMS != 5000 || status.SourceEndpoint != "newer.example:808" {
t.Fatalf("online status should keep newer received frame: %#v", status)
}
}
func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
err := repo.FastUpdateBatch(ctx, []envelope.FrameEnvelope{
result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{
{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
@@ -444,6 +643,10 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
},
},
{
Protocol: envelope.ProtocolJT808,
@@ -452,12 +655,17 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
ReceivedAtMS: 2200,
MessageID: "0x0200",
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}},
ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2},
Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2},
},
})
if err != nil {
t.Fatalf("FastUpdateBatch() error = %v", err)
}
if result.FieldsSeen != 4 || result.FieldsWritten != 4 || result.FieldsSkippedStale != 0 ||
result.EnvelopesSeen != 2 || result.EnvelopesUpdated != 2 {
t.Fatalf("FastUpdateBatchWithResult() result = %#v, want fields 4/4/0 and envelopes 2/2", result)
}
gbValues, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Result()
if err != nil {
@@ -481,6 +689,55 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
}
}
func TestRepositoryFastUpdateReportsSkippedEnvelopesWithoutWritingRedis(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{
{
Protocol: envelope.ProtocolGB32960,
MessageID: "0x01",
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1000,
Parsed: map[string]any{"platform_login": map[string]any{"username": "Hyundai"}},
},
{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
EventTimeMS: 2000,
ReceivedAtMS: 2000,
ParsedFields: map[string]any{"jt808.location.speed_kmh": 30},
},
{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN002",
EventTimeMS: 3000,
ReceivedAtMS: 3000,
Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}},
},
})
if err != nil {
t.Fatalf("FastUpdateBatchWithResult() error = %v", err)
}
if result.EnvelopesSeen != 3 || result.EnvelopesUpdated != 0 ||
result.EnvelopesSkippedNonRealtime != 1 || result.EnvelopesSkippedMissingVIN != 1 ||
result.EnvelopesSkippedMissingFields != 1 {
t.Fatalf("result = %#v, want non-realtime, missing vin and missing parsed fields skips", result)
}
if result.FieldsSeen != 0 || result.FieldsWritten != 0 || result.FieldsSkippedStale != 0 {
t.Fatalf("field result = %#v, want no field writes", result)
}
if repo.client.Exists(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val() != 0 {
t.Fatal("non-realtime frame should not write realtime kv")
}
if repo.client.Exists(ctx, "vehicle:online:JT808:VIN002").Val() != 0 {
t.Fatal("missing parsed fields frame should not mark vehicle online")
}
}
func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
@@ -495,6 +752,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0},
},
{
Protocol: envelope.ProtocolJT808,
@@ -503,6 +761,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
ReceivedAtMS: 2200,
MessageID: "0x0200",
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}},
ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2},
Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2},
},
} {
@@ -548,6 +807,7 @@ func TestOnlineIndexHandlerReturnsPagedOnlineStatuses(t *testing.T) {
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
}
@@ -579,6 +839,7 @@ func TestPipelineDebugHandlerReturnsProtocolSummary(t *testing.T) {
EventTimeMS: 3000,
ReceivedAtMS: 3300,
Parsed: map[string]any{"data": map[string]any{"TOTAL_MILEAGE": 1}},
ParsedFields: map[string]any{"yutong_mqtt.data.total_mileage": 1},
Fields: map[string]any{envelope.FieldTotalMileageKM: 1},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
@@ -728,6 +989,82 @@ func TestRepositoryUsesEnvelopeParsedFieldsForRealtimeKV(t *testing.T) {
}
}
func TestRepositoryDropsNonPositiveTotalMileageFromRealtimeKV(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
MessageID: "0x0200",
ParsedFields: map[string]any{
"jt808.location.latitude": "31.259555",
"jt808.location.longitude": "119.892413",
"jt808.location.total_mileage_km": "0",
},
Fields: map[string]any{
envelope.FieldLatitude: 31.259555,
envelope.FieldLongitude: 119.892413,
envelope.FieldTotalMileageKM: 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("HGetAll() error = %v", err)
}
if _, exists := values["jt808.location.total_mileage_km"]; exists {
t.Fatalf("realtime kv should drop non-positive mileage: %#v", values)
}
if values["jt808.location.latitude"] != "31.259555" {
t.Fatalf("realtime kv should keep valid fields: %#v", values)
}
}
func TestRepositoryDropsNonPositiveRawTotalMileageFromRealtimeKV(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
MessageID: "MQTT",
Parsed: map[string]any{
"data": map[string]any{
"LATITUDE": 30.590921,
"LONGITUDE": 121.075044,
"TOTAL_MILEAGE": 0,
},
},
ParsedFields: map[string]any{
"yutong_mqtt.data.latitude": 30.590921,
"yutong_mqtt.data.longitude": 121.075044,
"yutong_mqtt.data.total_mileage": 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolYutongMQTT, "LMRKH9AC3R1004101")).Result()
if err != nil {
t.Fatalf("HGetAll() error = %v", err)
}
if _, exists := values["yutong_mqtt.data.total_mileage"]; exists {
t.Fatalf("realtime kv should drop non-positive raw mileage: %#v", values)
}
if values["yutong_mqtt.data.latitude"] != "30.590921" {
t.Fatalf("realtime kv should keep valid fields: %#v", values)
}
}
func TestRepositorySkipsFramesWithoutVIN(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()