shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
go/vehicle-gateway/internal/realtime/kv.go
View File

@@ -8,6 +8,7 @@ import (
"strings"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)
type RealtimeKVField struct {
@@ -26,32 +27,36 @@ func BuildFieldsEnvelope(env envelope.FrameEnvelope) (envelope.FrameEnvelope, bo
if !envelope.IsRealtimeTelemetryFrame(env) {
return envelope.FrameEnvelope{}, false
}
fields, fieldTypes, ok := ParsedFieldsForEnvelope(env)
fields, _, ok := ParsedFieldsForEnvelope(env)
if !ok {
return envelope.FrameEnvelope{}, false
}
filterInvalidRealtimeMeasurementFields(fields)
if len(fields) == 0 || !telemetry.HasRealtimeFields(env.Protocol, fields) {
return envelope.FrameEnvelope{}, false
}
out := envelope.FrameEnvelope{
EventID: env.StableEventID() + ":fields",
TraceID: env.TraceID,
Protocol: env.Protocol,
MessageID: env.MessageID,
Sequence: env.Sequence,
VIN: env.VIN,
VehicleKeyHint: env.VehicleKeyHint,
Phone: env.Phone,
DeviceID: env.DeviceID,
Plate: env.Plate,
SourceEndpoint: env.SourceEndpoint,
EventTimeMS: env.EventTimeMS,
ReceivedAtMS: env.ReceivedAtMS,
Fields: fields,
ParsedFields: fields,
ParsedFieldTypes: fieldTypes,
Parsed: map[string]any{
"source_event_id": env.StableEventID(),
"field_mapping": realtimeFieldMappingVersion,
},
ParseStatus: envelope.ParseOK,
EventID: env.StableEventID() + ":fields",
TraceID: env.TraceID,
EventKind: envelope.EventKindFields,
SourceEventID: env.StableEventID(),
FieldMapping: realtimeFieldMappingVersion,
Protocol: env.Protocol,
MessageID: env.MessageID,
Sequence: env.Sequence,
VIN: env.VIN,
VehicleKeyHint: env.VehicleKeyHint,
Phone: env.Phone,
DeviceID: env.DeviceID,
Plate: env.Plate,
SourceEndpoint: env.SourceEndpoint,
SourceCode: env.SourceCode,
PlatformName: env.PlatformName,
SourceKind: env.SourceKind,
EventTimeMS: env.EventTimeMS,
ReceivedAtMS: env.ReceivedAtMS,
Fields: fields,
ParseStatus: envelope.ParseOK,
}
return out, true
}
@@ -61,12 +66,34 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool {
return false
}
if len(env.ParsedFields) > 0 {
if derived, derivedTypes, ok := computeParsedFieldsFromParsed(*env); ok {
for field, value := range derived {
if _, exists := env.ParsedFields[field]; !exists {
env.ParsedFields[field] = value
}
}
if env.ParsedFieldTypes == nil {
env.ParsedFieldTypes = map[string]string{}
}
for field, valueType := range derivedTypes {
if _, exists := env.ParsedFieldTypes[field]; !exists {
env.ParsedFieldTypes[field] = valueType
}
}
}
inferredTypes := inferParsedFieldTypes(env.ParsedFields)
if env.ParsedFieldTypes == nil {
env.ParsedFieldTypes = inferParsedFieldTypes(env.ParsedFields)
env.ParsedFieldTypes = inferredTypes
} else {
for field, valueType := range inferredTypes {
if _, exists := env.ParsedFieldTypes[field]; !exists {
env.ParsedFieldTypes[field] = valueType
}
}
}
return true
}
fields, fieldTypes, ok := ParsedFieldsForEnvelope(*env)
fields, fieldTypes, ok := ComputeParsedFieldsForEnvelope(*env)
if !ok {
return false
}
@@ -76,14 +103,27 @@ func EnsureParsedFields(env *envelope.FrameEnvelope) bool {
}
func ParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
if len(env.ParsedFields) > 0 {
fields := cloneAnyMap(env.ParsedFields)
fieldTypes := cloneStringMap(env.ParsedFieldTypes)
if len(fieldTypes) == 0 {
fieldTypes = inferParsedFieldTypes(fields)
}
if len(env.ParsedFields) == 0 {
return nil, nil, false
}
fields := cloneAnyMap(env.ParsedFields)
fieldTypes := cloneStringMap(env.ParsedFieldTypes)
if len(fieldTypes) == 0 {
fieldTypes = inferParsedFieldTypes(fields)
}
return fields, fieldTypes, true
}
// ComputeParsedFieldsForEnvelope is reserved for ingress and offline backfills.
// Runtime projections must consume the precomputed ParsedFields contract instead.
func ComputeParsedFieldsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
if fields, fieldTypes, ok := ParsedFieldsForEnvelope(env); ok {
return fields, fieldTypes, true
}
return computeParsedFieldsFromParsed(env)
}
func computeParsedFieldsFromParsed(env envelope.FrameEnvelope) (map[string]any, map[string]string, bool) {
rows := realtimeKVFields(env, env.Parsed)
if len(rows) == 0 {
return nil, nil, false
@@ -119,6 +159,22 @@ func inferParsedFieldTypes(fields map[string]any) map[string]string {
return out
}
func filterInvalidRealtimeMeasurementFields(fields map[string]any) {
for field, value := range fields {
if isRealtimeTotalMileageField(field) && !positiveNumber(value) {
delete(fields, field)
}
}
}
func isRealtimeTotalMileageField(field string) bool {
field = strings.ToLower(strings.ReplaceAll(strings.TrimSpace(field), "/", "."))
return field == envelope.FieldTotalMileageKM ||
field == "total_mileage" ||
strings.HasSuffix(field, "."+envelope.FieldTotalMileageKM) ||
strings.HasSuffix(field, ".total_mileage")
}
func cloneAnyMap(in map[string]any) map[string]any {
if len(in) == 0 {
return nil
@@ -191,8 +247,9 @@ const realtimeFieldMappingVersion = "2026-07-03.v1"
func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []RealtimeKVField {
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
if env.Protocol == envelope.ProtocolGB32960 && len(parsed) > 0 {
parsed = cloneMap(parsed)
normalizeGB32960RealtimeParsed(parsed)
}
eventID := env.StableEventID()
mapping := realtimeMapping(env.Protocol)
@@ -205,6 +262,9 @@ func realtimeKVFields(env envelope.FrameEnvelope, parsed map[string]any) []Realt
}
sort.Strings(names)
for _, name := range names {
if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) {
continue
}
value, valueType, ok := stringifyKVValue(flat[name])
if !ok {
continue
@@ -255,6 +315,9 @@ func gb32960KVFields(env envelope.FrameEnvelope, parsed map[string]any, mapping
}
sort.Strings(names)
for _, name := range names {
if isRealtimeTotalMileageField(realtimeKVFieldPath(domain, name)) && !positiveNumber(flat[name]) {
continue
}
value, valueType, ok := stringifyKVValue(flat[name])
if !ok {
continue
@@ -335,6 +398,12 @@ func flattenKV(prefix string, value any, out map[string]any, mapping protocolFie
}
flattenKV(next, typed[key], out, mapping)
}
case map[string]bool:
normalized := make(map[string]any, len(typed))
for key, item := range typed {
normalized[key] = item
}
flattenKV(prefix, normalized, out, mapping)
case []any:
for index, item := range typed {
itemMap, ok := item.(map[string]any)
@@ -397,8 +466,14 @@ func mappedTopLevelDomain(mapping protocolFieldMapping, key string) string {
if key == "" {
return ""
}
name := mapping.TopLevelName[key]
if name == "" {
name := ""
if len(mapping.TopLevelName) > 0 {
var ok bool
name, ok = mapping.TopLevelName[key]
if !ok {
return ""
}
} else {
name = sanitizeFieldPart(key)
}
if name == "" {

258
go/vehicle-gateway/internal/realtime/kv_test.go
View File

@@ -1,6 +1,8 @@
package realtime
import (
"encoding/json"
"strings"
"testing"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
@@ -40,7 +42,7 @@ func TestRealtimeKVFieldsFromGB32960ParsedDomains(t *testing.T) {
}
func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
MessageID: "0x02",
@@ -61,10 +63,17 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
}},
},
},
})
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
if len(fieldsEnv.ParsedFields) != 0 || len(fieldsEnv.ParsedFieldTypes) != 0 || len(fieldsEnv.Parsed) != 0 {
t.Fatalf("fields envelope should keep only slim fields payload, parsed=%#v parsed_fields=%#v parsed_field_types=%#v", fieldsEnv.Parsed, fieldsEnv.ParsedFields, fieldsEnv.ParsedFieldTypes)
}
for _, bareKey := range []string{"charge_status", "soc_percent", "total_mileage_km"} {
if _, exists := fieldsEnv.Fields[bareKey]; exists {
t.Fatalf("fields envelope should not expose non-protocol bare key %q: %#v", bareKey, fieldsEnv.Fields)
@@ -81,6 +90,240 @@ func TestBuildFieldsEnvelopeUsesOnlyProtocolMappedFieldNames(t *testing.T) {
}
}
func TestEnsureParsedFieldsKeepsUnknownVINProtocolFieldsAndExcludesAnnotations(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Phone: "13307795425",
Parsed: map[string]any{
"registration": map[string]any{
"manufacturer": "YUTNG",
},
"identity": map[string]any{
"resolved": false,
"reason": "no_binding",
},
},
ParsedFields: map[string]any{
"jt808.location.speed_kmh": "12.3",
},
}
if !EnsureParsedFields(&env) {
t.Fatal("unknown VIN raw frame should still retain parsed fields")
}
if got := env.ParsedFields["jt808.location.speed_kmh"]; got != "12.3" {
t.Fatalf("precomputed field = %#v", got)
}
if got := env.ParsedFields["jt808.registration.manufacturer"]; got != "YUTNG" {
t.Fatalf("merged registration field = %#v", got)
}
if _, exists := env.ParsedFields["jt808.identity.resolved"]; exists {
t.Fatalf("derived identity annotations must not enter protocol fields: %#v", env.ParsedFields)
}
for _, field := range []string{"jt808.location.speed_kmh", "jt808.registration.manufacturer"} {
if env.ParsedFieldTypes[field] == "" {
t.Fatalf("field type missing for %s: %#v", field, env.ParsedFieldTypes)
}
}
}
func TestBuildFieldsEnvelopeCopiesSourceMetadata(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
MessageID: "MQTT",
SourceEndpoint: "mqtt://yutong/ytforward/shln/1",
SourceCode: "yutong",
PlatformName: "宇通",
SourceKind: "PLATFORM",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-yutong",
Fields: map[string]any{
envelope.FieldTotalMileageKM: 56905,
},
Parsed: map[string]any{
"data": map[string]any{"TOTAL_MILEAGE": 56905000},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
if fieldsEnv.SourceCode != "yutong" || fieldsEnv.PlatformName != "宇通" || fieldsEnv.SourceKind != "PLATFORM" {
t.Fatalf("source metadata = code:%q platform:%q kind:%q", fieldsEnv.SourceCode, fieldsEnv.PlatformName, fieldsEnv.SourceKind)
}
}
func TestBuildFieldsEnvelopeDropsNonPositiveTotalMileage(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-1",
ParsedFields: map[string]any{
"jt808.location.latitude": "31.259555",
"jt808.location.longitude": "119.892413",
"jt808.location.total_mileage_km": "0",
},
})
if !ok {
t.Fatal("BuildFieldsEnvelope() should still emit valid fields")
}
if _, exists := fieldsEnv.Fields["jt808.location.total_mileage_km"]; exists {
t.Fatalf("fields envelope should drop non-positive mileage: %#v", fieldsEnv.Fields)
}
if fieldsEnv.Fields["jt808.location.latitude"] != "31.259555" {
t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields)
}
}
func TestBuildFieldsEnvelopeDropsNonPositiveRawTotalMileage(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
MessageID: "MQTT",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-yutong-zero-mileage",
Parsed: map[string]any{
"data": map[string]any{
"LATITUDE": 30.590921,
"LONGITUDE": 121.075044,
"TOTAL_MILEAGE": 0,
},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should still emit non-mileage fields")
}
if _, exists := fieldsEnv.Fields["yutong_mqtt.data.total_mileage"]; exists {
t.Fatalf("fields envelope should drop non-positive raw mileage: %#v", fieldsEnv.Fields)
}
if fieldsEnv.Fields["yutong_mqtt.data.latitude"] != "30.590921" {
t.Fatalf("fields envelope should keep valid location fields: %#v", fieldsEnv.Fields)
}
}
func TestBuildFieldsEnvelopeJSONOmitsDuplicateParsedPayload(t *testing.T) {
fieldsEnv, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
EventID: "event-1",
ParsedFields: map[string]any{
"jt808.location.total_mileage_km": "10241.2",
"jt808.location.speed_kmh": "23",
},
ParsedFieldTypes: map[string]string{
"jt808.location.total_mileage_km": "number",
"jt808.location.speed_kmh": "number",
},
})
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
data, err := json.Marshal(fieldsEnv)
if err != nil {
t.Fatalf("Marshal(fieldsEnv) error = %v", err)
}
text := string(data)
for _, duplicateKey := range []string{`"parsed_fields"`, `"parsed_field_types"`, `"parsed"`} {
if strings.Contains(text, duplicateKey) {
t.Fatalf("fields JSON should omit duplicate %s payload: %s", duplicateKey, text)
}
}
if !strings.Contains(text, `"fields"`) || !strings.Contains(text, `"source_event_id"`) || !strings.Contains(text, `"field_mapping"`) {
t.Fatalf("fields JSON missing slim payload metadata: %s", text)
}
}
func TestBuildFieldsEnvelopeOmitsGB32960VendorFragmentFields(t *testing.T) {
env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
MessageID: "0x02",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Parsed: map[string]any{
"data_units": []any{
map[string]any{
"type": "0x30",
"name": "gd_fc_stack",
"value": map[string]any{
"stack_count": 1,
"summaries": []any{
map[string]any{
"cell_count": 432,
"stack_water_outlet_temp_c": 63,
"frame_cell_start": 401,
"frame_cell_count": 32,
"frame_max_cell_voltage_v": 1,
"frame_min_cell_voltage_v": 1,
"hydrogen_inlet_pressure_kpa": 130,
"air_inlet_pressure_kpa": 150,
"stack_water_outlet_temp_extra": "kept",
},
},
},
},
},
},
}
if !EnsureParsedFields(&env) {
t.Fatal("EnsureParsedFields() should compute ingress fields")
}
fieldsEnv, ok := BuildFieldsEnvelope(env)
if !ok {
t.Fatal("BuildFieldsEnvelope() should emit mapped fields")
}
for _, fragmentField := range []string{
"gb32960.gd_fc_stack.frame_cell_start",
"gb32960.gd_fc_stack.frame_cell_count",
"gb32960.gd_fc_stack.frame_max_cell_voltage_v",
"gb32960.gd_fc_stack.frame_min_cell_voltage_v",
} {
if _, exists := fieldsEnv.Fields[fragmentField]; exists {
t.Fatalf("fields envelope should omit fragment-only field %q: %#v", fragmentField, fieldsEnv.Fields)
}
}
for _, keptField := range []string{
"gb32960.gd_fc_stack.stack_count",
"gb32960.gd_fc_stack.stack_water_outlet_temp_c",
"gb32960.gd_fc_stack.hydrogen_inlet_pressure_kpa",
} {
if _, exists := fieldsEnv.Fields[keptField]; !exists {
t.Fatalf("fields envelope should keep current-state field %q: %#v", keptField, fieldsEnv.Fields)
}
}
}
func TestBuildFieldsEnvelopeRequiresIngressParsedFields(t *testing.T) {
_, ok := BuildFieldsEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
MessageID: "0x0200",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
Parsed: map[string]any{
"location": map[string]any{"speed_kmh": 99},
},
})
if ok {
t.Fatal("BuildFieldsEnvelope() must not re-flatten parsed payload downstream")
}
}
func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
jtRows := realtimeKVFields(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
@@ -97,6 +340,10 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
"latitude": 30.2,
"total_mileage_km": 10241.2,
"additional": []any{map[string]any{"id": "0x01", "value_hex": "00077235"}},
"io_status": map[string]any{
"value": uint16(2),
"bits": map[string]bool{"deep_sleep": false, "sleep": true},
},
},
})
jtValues := kvMap(jtRows)
@@ -105,9 +352,14 @@ func TestRealtimeKVFieldsFromJT808ParsedFields(t *testing.T) {
}
if jtValues["jt808.location/total_mileage_km"] != "10241.2" ||
jtValues["jt808.location/longitude"] != "121.1" ||
jtValues["jt808.location/additional.additional_1.id"] != "0x01" {
jtValues["jt808.location/additional.additional_1.id"] != "0x01" ||
jtValues["jt808.location/io_status.bits.deep_sleep"] != "false" ||
jtValues["jt808.location/io_status.bits.sleep"] != "true" {
t.Fatalf("jt808 location kv missing: %#v", jtValues)
}
if _, exists := jtValues["jt808.location/io_status.bits"]; exists {
t.Fatalf("JT808 bit fields must be flattened instead of embedded JSON: %#v", jtValues)
}
if jtValues["jt808.location/soc_percent"] != "" {
t.Fatalf("jt808 kv should not include standardized env.Fields-only values: %#v", jtValues)
}

7
go/vehicle-gateway/internal/realtime/mysql_query.go
View File

@@ -46,6 +46,7 @@ type LocationRow struct {
Longitude float64 `json:"longitude"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
TotalMileageKM *float64 `json:"total_mileage_km,omitempty"`
TotalMileageAt string `json:"total_mileage_event_time,omitempty"`
SOCPercent *float64 `json:"soc_percent,omitempty"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
DirectionDeg *float64 `json:"direction_deg,omitempty"`
@@ -124,7 +125,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
query = normalizeRealtimeTableQuery(query)
sqlText, args := buildRealtimeSelectSQL(
"vehicle_realtime_location",
"protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at",
"protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at",
query,
)
rows, err := r.db.QueryContext(ctx, sqlText, args...)
@@ -136,7 +137,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
out := make([]LocationRow, 0)
for rows.Next() {
var row LocationRow
var eventTime, receivedAt, updatedAt scanSQLDateTime
var eventTime, mileageAt, receivedAt, updatedAt scanSQLDateTime
var speed, mileage, soc, altitude, direction sql.NullFloat64
var alarm, status sql.NullInt64
if err := rows.Scan(
@@ -148,6 +149,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
&row.Longitude,
&speed,
&mileage,
&mileageAt,
&soc,
&altitude,
&direction,
@@ -162,6 +164,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
row.EventTime = eventTime.String
row.SpeedKMH = nullableFloat(speed)
row.TotalMileageKM = nullableFloat(mileage)
row.TotalMileageAt = mileageAt.String
row.SOCPercent = nullableFloat(soc)
row.AltitudeM = nullableFloat(altitude)
row.DirectionDeg = nullableFloat(direction)

14
go/vehicle-gateway/internal/realtime/mysql_query_test.go
View File

@@ -61,14 +61,14 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_location").
WithArgs("JT808", "粤B98765").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1))
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location").
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location").
WithArgs("JT808", "粤B98765", 10, 10).
WillReturnRows(sqlmock.NewRows([]string{
"protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km",
"soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at",
"total_mileage_event_time", "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at",
}).AddRow(
"JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9,
nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04",
"2026-07-02 16:11:02.000", nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04",
))
handler := NewLocationQueryHandler(NewLocationQueryRepository(db))
@@ -81,7 +81,7 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"protocol":"JT808"`, `"latitude":30.123456`, `"longitude":120.654321`, `"total_mileage_km":48798.9`, `"offset":10`} {
for _, want := range []string{`"protocol":"JT808"`, `"latitude":30.123456`, `"longitude":120.654321`, `"total_mileage_km":48798.9`, `"total_mileage_event_time":"2026-07-02 16:11:02.000"`, `"offset":10`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
@@ -133,14 +133,14 @@ func TestLocationQueryHandlerSkipsTotalCountByDefault(t *testing.T) {
}
defer db.Close()
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location").
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location").
WithArgs("JT808", 1, 0).
WillReturnRows(sqlmock.NewRows([]string{
"protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km",
"soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at",
"total_mileage_event_time", "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at",
}).AddRow(
"JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9,
nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04",
"2026-07-02 16:11:02.000", nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04",
))
handler := NewLocationQueryHandler(NewLocationQueryRepository(db))

736
go/vehicle-gateway/internal/realtime/openapi.go
View File

@@ -79,11 +79,358 @@ func realtimeOpenAPISpec() map[string]any {
},
},
},
"/api/stats/daily-metrics": map[string]any{
"get": map[string]any{
"summary": "查询车辆每日里程",
"description": "从 vehicle_daily_mileage 查询最终选举后的每日里程，并通过 source_id 关联 vehicle_data_source 返回来源证据。默认不执行 COUNT(*)，total 默认表示本页返回条数；需要精确总数时传 includeTotal=true。",
"tags": []string{"Stats API"},
"x-table": "vehicle_daily_mileage,vehicle_data_source",
"parameters": dailyMetricParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/sources": map[string]any{
"get": map[string]any{
"summary": "查询每日里程候选来源",
"description": "从 vehicle_daily_mileage_source 查询每辆车、每天、每个协议下各来源独立计算出的里程候选，并关联 vehicle_data_source 返回平台、来源类型和可信优先级，用于解释最终日里程为什么选中某个来源。",
"tags": []string{"Stats API"},
"x-table": "vehicle_daily_mileage_source,vehicle_data_source",
"parameters": dailyMetricSourceParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric source page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourcePage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/sources/quality": map[string]any{
"get": map[string]any{
"summary": "汇总每日里程候选质量",
"description": "按日期、协议、quality_status、quality_reason 汇总 vehicle_daily_mileage_source，用于发现某协议或某来源类型是否正在批量产生无基线、异常跳变等候选里程质量问题。",
"tags": []string{"Stats API"},
"x-table": "vehicle_daily_mileage_source",
"parameters": dailyMetricSourceParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric source quality summary page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourceQualityPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/sources/selection": map[string]any{
"get": map[string]any{
"summary": "汇总每日里程来源选举",
"description": "按日期、协议、selection_status、selection_reason 汇总 vehicle_daily_mileage_source，并结合 vehicle_data_source 解释各来源被选中、质量淘汰、禁用淘汰或低优先级未选中的原因。",
"tags": []string{"Stats API"},
"x-table": "vehicle_daily_mileage_source,vehicle_data_source",
"parameters": dailyMetricSourceParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric source selection summary page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricSourceSelectionPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/diagnostics": map[string]any{
"get": map[string]any{
"summary": "诊断实时在线但日里程缺失",
"description": "从 vehicle_realtime_snapshot/location 按 event_time/received_at 找到指定日期活跃车辆，并关联 vehicle_daily_mileage 与 vehicle_daily_mileage_source，解释车辆有实时数据但日里程缺失的原因。默认不执行 COUNT(*)，total 默认表示本页返回条数；需要精确总数时传 includeTotal=true。",
"tags": []string{"Stats API"},
"x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source",
"parameters": dailyMetricDiagnosisParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric diagnostics page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/diagnostics/summary": map[string]any{
"get": map[string]any{
"summary": "汇总每日里程诊断",
"description": "按协议汇总指定日期 event_time/received_at 活跃车辆的日里程诊断结果，返回 OK、MISSING_DAILY、NO_SOURCE_SAMPLE、NO_TOTAL_MILEAGE 等分类数量，用于大屏、告警和排障入口。",
"tags": []string{"Stats API"},
"x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source",
"parameters": dailyMetricDiagnosisSummaryParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric diagnostics summary",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisSummaryPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/diagnostics/reasons": map[string]any{
"get": map[string]any{
"summary": "按原因汇总每日里程诊断",
"description": "按协议、诊断分类和 reason 汇总指定日期 event_time/received_at 活跃车辆的日里程诊断结果，用于大屏和告警直接区分源头未上报、总里程为 0、旧总里程未更新、统计抽取缺失等问题。",
"tags": []string{"Stats API"},
"x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source",
"parameters": dailyMetricDiagnosisReasonSummaryParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric diagnostics reason summary",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisReasonSummaryPage"},
},
},
},
},
},
},
"/api/stats/daily-metrics/diagnostics/field-status": map[string]any{
"get": map[string]any{
"summary": "按字段状态汇总每日里程诊断",
"description": "按协议、诊断分类、reason 和实时总里程字段状态汇总，直接区分源头没有可用总里程、疑似字段未映射、标准总里程为 0、标准总里程停留在旧日期、统计消费缺失等问题。",
"tags": []string{"Stats API"},
"x-table": "vehicle_realtime_snapshot,vehicle_realtime_location,vehicle_daily_mileage,vehicle_daily_mileage_source",
"parameters": dailyMetricDiagnosisReasonSummaryParameters(),
"responses": map[string]any{
"200": map[string]any{
"description": "Daily metric diagnostics field status summary",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisFieldStatusSummaryPage"},
},
},
},
},
},
},
"/api/stats/data-sources": map[string]any{
"get": map[string]any{
"summary": "查询数据来源",
"description": "查询 vehicle_data_source 中自动发现的协议来源，用于人工维护平台名称、source_code、可信优先级和启停状态。",
"tags": []string{"Data Source API"},
"x-table": "vehicle_data_source",
"parameters": []map[string]any{
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},
{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceKind", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"UNKNOWN", "PLATFORM", "DIRECT"}}, "required": false},
{"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false},
{"name": "enabled", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
},
"responses": map[string]any{
"200": map[string]any{
"description": "Data source page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DataSourcePage"},
},
},
},
},
},
},
"/api/stats/data-sources/diagnostics": map[string]any{
"get": map[string]any{
"summary": "诊断 JT808 来源映射",
"description": "按来源 IP 汇总 jt808_registration 与 vehicle_identifier 的匹配情况，帮助判断缺失 source_code 的 808 来源应维护到哪个平台。",
"tags": []string{"Data Source API"},
"x-table": "vehicle_data_source,jt808_registration,vehicle_identifier",
"parameters": []map[string]any{
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"JT808"}, "default": "JT808"}, "required": false},
{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean", "default": true}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
},
"responses": map[string]any{
"200": map[string]any{
"description": "Data source diagnostics page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DataSourceDiagnosticsPage"},
},
},
},
},
},
},
"/api/stats/data-sources/kind-suggestions": map[string]any{
"get": map[string]any{
"summary": "建议 JT808 来源类型",
"description": "基于来源活跃时长、注册手机号、vehicle_identifier 匹配和 source_code 情况，对 UNKNOWN 来源给出 PLATFORM/DIRECT/UNKNOWN 的只读建议。",
"tags": []string{"Data Source API"},
"x-table": "vehicle_data_source,jt808_registration,vehicle_identifier",
"parameters": []map[string]any{
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"JT808"}, "default": "JT808"}, "required": false},
{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceKind", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"UNKNOWN", "PLATFORM", "DIRECT"}, "default": "UNKNOWN"}, "required": false},
{"name": "sourceCodeMissing", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
},
"responses": map[string]any{
"200": map[string]any{
"description": "Data source kind suggestion page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DataSourceDiagnosticsPage"},
},
},
},
},
},
},
"/api/stats/data-sources/jt808-identity-gaps": map[string]any{
"get": map[string]any{
"summary": "诊断 JT808 未绑定设备",
"description": "列出 jt808_registration 中仍无法通过 phone/plate 关联 VIN 的设备，用于定位 0x0200 no_binding 和补充 vehicle_identifier 映射。",
"tags": []string{"Data Source API"},
"x-table": "jt808_registration,vehicle_identifier,vehicle_identity_binding,vehicle_data_source",
"parameters": []map[string]any{
{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "recentSeconds", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 2592000, "default": 86400}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
},
"responses": map[string]any{
"200": map[string]any{
"description": "JT808 identity gap page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/JT808IdentityGapPage"},
},
},
},
},
},
},
"/api/stats/data-sources/jt808-mapping-gaps": map[string]any{
"get": map[string]any{
"summary": "诊断 JT808 来源映射缺口",
"description": "列出已配置来源平台但缺少当前 source_code 下 JT808_PHONE 到 VIN 映射的注册手机号，用于维护 vehicle_identifier 并提升 VIN 解析、在线统计和里程来源选举覆盖率。",
"tags": []string{"Data Source API"},
"x-table": "jt808_registration,vehicle_identifier,vehicle_data_source",
"parameters": []map[string]any{
{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "sourceCode", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "recentSeconds", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 2592000, "default": 86400}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
},
"responses": map[string]any{
"200": map[string]any{
"description": "JT808 mapping gap page",
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/JT808MappingGapPage"},
},
},
},
},
},
},
"/api/stats/data-sources/{id}": map[string]any{
"patch": map[string]any{
"summary": "维护数据来源人工字段",
"description": "只允许更新 platform_name、source_code、source_kind、trust_priority、enabled、remark；source_ip、latest_seen_at 等运行态字段由接入链路自动维护。",
"tags": []string{"Data Source API"},
"parameters": []map[string]any{
{"name": "id", "in": "path", "schema": map[string]any{"type": "integer"}, "required": true},
},
"requestBody": map[string]any{
"required": true,
"content": map[string]any{
"application/json": map[string]any{
"schema": map[string]any{"$ref": "#/components/schemas/DataSourceUpdate"},
},
},
},
"responses": map[string]any{
"200": map[string]any{"description": "Updated"},
},
},
},
},
"components": map[string]any{
"schemas": map[string]any{
"SnapshotPage": pageSchema("#/components/schemas/SnapshotRow"),
"LocationPage": pageSchema("#/components/schemas/LocationRow"),
"SnapshotPage": pageSchema("#/components/schemas/SnapshotRow"),
"LocationPage": pageSchema("#/components/schemas/LocationRow"),
"DailyMetricPage": pageSchema("#/components/schemas/DailyMetricRow"),
"DailyMetricSourcePage": pageSchema("#/components/schemas/DailyMetricSourceRow"),
"DailyMetricSourceQualityPage": pageSchema("#/components/schemas/DailyMetricSourceQualityRow"),
"DailyMetricSourceSelectionPage": pageSchema("#/components/schemas/DailyMetricSourceSelectionRow"),
"DailyMetricDiagnosisPage": pageSchema("#/components/schemas/DailyMetricDiagnosisRow"),
"DailyMetricDiagnosisSummaryPage": map[string]any{
"type": "object",
"properties": map[string]any{
"items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisSummaryRow"}},
"total": integerSchema(3),
"active_total": integerSchema(256),
"actionable_issue_total": integerSchema(6),
},
},
"DailyMetricDiagnosisReasonSummaryPage": map[string]any{
"type": "object",
"properties": map[string]any{
"items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisReasonSummaryRow"}},
"total": integerSchema(5),
"vehicle_total": integerSchema(347),
"actionable_issue_total": integerSchema(9),
"pipeline_issue_total": integerSchema(0),
"source_data_issue_total": integerSchema(9),
},
},
"DailyMetricDiagnosisFieldStatusSummaryPage": map[string]any{
"type": "object",
"properties": map[string]any{
"items": map[string]any{"type": "array", "items": map[string]any{"$ref": "#/components/schemas/DailyMetricDiagnosisFieldStatusSummaryRow"}},
"total": integerSchema(7),
"vehicle_total": integerSchema(347),
"actionable_issue_total": integerSchema(9),
"pipeline_issue_total": integerSchema(2),
"source_data_issue_total": integerSchema(7),
},
},
"DataSourcePage": pageSchema("#/components/schemas/DataSourceRow"),
"DataSourceDiagnosticsPage": pageSchema("#/components/schemas/DataSourceDiagnosticRow"),
"JT808IdentityGapPage": pageSchema("#/components/schemas/JT808IdentityGapRow"),
"JT808MappingGapPage": pageSchema("#/components/schemas/JT808MappingGapRow"),
"SnapshotRow": map[string]any{
"type": "object",
"properties": map[string]any{
@@ -99,22 +446,291 @@ func realtimeOpenAPISpec() map[string]any {
"LocationRow": map[string]any{
"type": "object",
"properties": map[string]any{
"protocol": stringSchema("JT808"),
"vin": stringSchema("LKLG7C4E3NA774736"),
"plate": stringSchema("粤B98765"),
"event_time": stringSchema("2026-07-02 16:11:02.000"),
"latitude": numberSchema(30.123456),
"longitude": numberSchema(120.654321),
"speed_kmh": numberSchema(54.3),
"total_mileage_km": numberSchema(48798.9),
"total_mileage_event_time": stringSchema("2026-07-02 16:11:02.000"),
"soc_percent": numberSchema(81.5),
"altitude_m": numberSchema(19.0),
"direction_deg": numberSchema(88.0),
"alarm_flag": integerSchema(0),
"status_flag": integerSchema(3),
"received_at": stringSchema("2026-07-02 16:11:03.000"),
"event_id": stringSchema("event id"),
"updated_at": stringSchema("2026-07-02 16:11:04"),
},
},
"DailyMetricRow": map[string]any{
"type": "object",
"properties": map[string]any{
"vin": stringSchema("LA9GG64L7PBAF4001"),
"stat_date": stringSchema("2026-07-08"),
"protocol": stringSchema("JT808"),
"source_id": integerSchema(3),
"source_ip": stringSchema("115.231.168.135"),
"latest_source_endpoint": stringSchema("115.231.168.135:41561"),
"platform_name": stringSchema("G7 平台"),
"source_code": stringSchema("G7S"),
"source_kind": stringSchema("PLATFORM"),
"daily_mileage_km": numberSchema(23.1),
"latest_total_mileage_km": numberSchema(4123.9),
"updated_at": stringSchema("2026-07-08 13:30:57"),
},
},
"DailyMetricSourceRow": map[string]any{
"type": "object",
"properties": map[string]any{
"vin": stringSchema("LA9GG64L7PBAF4001"),
"stat_date": stringSchema("2026-07-08"),
"protocol": stringSchema("JT808"),
"source_key": stringSchema("JT808:13307795425@115.231.168.135"),
"source_ip": stringSchema("115.231.168.135"),
"source_endpoint": stringSchema("115.231.168.135:41561"),
"phone": stringSchema("13307795425"),
"platform_name": stringSchema("信达"),
"source_id": integerSchema(5),
"source_code": stringSchema("xinda"),
"source_kind": stringSchema("PLATFORM"),
"source_enabled": map[string]any{"type": "boolean", "example": true},
"trust_priority": integerSchema(10),
"first_total_mileage_km": numberSchema(4100.8),
"latest_total_mileage_km": numberSchema(4123.9),
"daily_mileage_km": numberSchema(23.1),
"sample_count": integerSchema(128),
"first_event_time": stringSchema("2026-07-08 00:01:00"),
"latest_event_time": stringSchema("2026-07-08 23:59:00"),
"quality_status": stringSchema("OK"),
"quality_reason": stringSchema("historical_source_baseline"),
"is_selected": map[string]any{"type": "boolean", "example": true},
"selection_status": stringSchema("selected"),
"selection_reason": stringSchema("selected_current_projection"),
"selection_action": stringSchema("当前来源已被投影到最终日里程表"),
"updated_at": stringSchema("2026-07-08 23:59:10"),
},
},
"DailyMetricSourceQualityRow": map[string]any{
"type": "object",
"properties": map[string]any{
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("JT808"),
"vin": stringSchema("LKLG7C4E3NA774736"),
"plate": stringSchema("粤B98765"),
"event_time": stringSchema("2026-07-02 16:11:02.000"),
"latitude": numberSchema(30.123456),
"longitude": numberSchema(120.654321),
"speed_kmh": numberSchema(54.3),
"total_mileage_km": numberSchema(48798.9),
"soc_percent": numberSchema(81.5),
"altitude_m": numberSchema(19.0),
"direction_deg": numberSchema(88.0),
"alarm_flag": integerSchema(0),
"status_flag": integerSchema(3),
"received_at": stringSchema("2026-07-02 16:11:03.000"),
"event_id": stringSchema("event id"),
"updated_at": stringSchema("2026-07-02 16:11:04"),
"quality_status": stringSchema("INVALID_DELTA"),
"quality_reason": stringSchema("outside_daily_range"),
"source_count": integerSchema(3),
"vehicle_count": integerSchema(3),
"selected_count": integerSchema(0),
"sample_count": integerSchema(128),
"daily_mileage_km": numberSchema(12435.2),
},
},
"DailyMetricSourceSelectionRow": map[string]any{
"type": "object",
"properties": map[string]any{
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("JT808"),
"selection_status": stringSchema("not_selected"),
"selection_reason": stringSchema("lower_trust_priority_or_sample_count"),
"selection_action": stringSchema("来源质量可用，但被更高可信优先级、更多样本或更新时间更新的来源覆盖"),
"source_count": integerSchema(36),
"vehicle_count": integerSchema(31),
"selected_count": integerSchema(0),
"sample_count": integerSchema(3200),
"daily_mileage_km": numberSchema(812.5),
},
},
"DailyMetricDiagnosisRow": map[string]any{
"type": "object",
"properties": map[string]any{
"vin": stringSchema("LA9GG64L7PBAF4001"),
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("JT808"),
"plate": stringSchema("粤A12345"),
"platform_name": stringSchema("信达"),
"peer": stringSchema("115.231.168.135:47849"),
"snapshot_event_time": stringSchema("2026-07-12 10:01:02.000"),
"location_event_time": stringSchema("2026-07-12 10:01:02.000"),
"snapshot_updated_at": stringSchema("2026-07-12 10:01:03"),
"location_updated_at": stringSchema("2026-07-12 10:01:03"),
"realtime_total_mileage_km": numberSchema(25246.6),
"realtime_total_mileage_event_time": stringSchema("2026-07-12 10:01:02.000"),
"daily_mileage_km": numberSchema(0.3),
"daily_latest_total_mileage_km": numberSchema(25246.6),
"source_sample_count": integerSchema(36),
"ok_source_count": integerSchema(1),
"selectable_source_count": integerSchema(1),
"latest_stat_event_time": stringSchema("2026-07-12 10:01:02"),
"quality_statuses": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"OK"}},
"realtime_field_count": integerSchema(32),
"realtime_sample_fields": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"yutong_mqtt.data.latitude", "yutong_mqtt.data.longitude", "yutong_mqtt.data.meter_speed"}},
"realtime_mileage_field_status": map[string]any{"type": "string", "enum": dailyMetricMileageFieldStatusEnum(), "example": "no_candidate_mileage_field"},
"realtime_mileage_candidate_fields": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"yutong_mqtt.data.odometer"}},
"realtime_mileage_evidence": stringSchema("实时快照已有字段，但没有发现 mileage/odometer/odo 等疑似总里程字段"),
"diagnosis": stringSchema("OK"),
"reason": stringSchema("daily_metric_exists"),
"severity": stringSchema("ok"),
},
},
"DailyMetricDiagnosisSummaryRow": map[string]any{
"type": "object",
"properties": map[string]any{
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("JT808"),
"active_count": integerSchema(244),
"ok_count": integerSchema(244),
"missing_daily_count": integerSchema(0),
"no_source_sample_count": integerSchema(0),
"no_total_mileage_count": integerSchema(0),
"actionable_issue_count": integerSchema(0),
},
},
"DailyMetricDiagnosisReasonSummaryRow": map[string]any{
"type": "object",
"properties": map[string]any{
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("YUTONG_MQTT"),
"diagnosis": stringSchema("NO_TOTAL_MILEAGE"),
"reason": stringSchema("realtime_total_mileage_not_reported_on_stat_date"),
"count": integerSchema(7),
"severity": stringSchema("source_data"),
},
},
"DailyMetricDiagnosisFieldStatusSummaryRow": map[string]any{
"type": "object",
"properties": map[string]any{
"stat_date": stringSchema("2026-07-12"),
"protocol": stringSchema("YUTONG_MQTT"),
"diagnosis": stringSchema("NO_TOTAL_MILEAGE"),
"reason": stringSchema("realtime_total_mileage_missing"),
"realtime_mileage_field_status": map[string]any{"type": "string", "enum": dailyMetricMileageFieldStatusEnum(), "example": "no_candidate_mileage_field"},
"count": integerSchema(7),
"severity": stringSchema("source_data"),
"field_status_severity": stringSchema("source_data"),
"recommended_operator_action": stringSchema("当日实时数据缺少总里程字段，核对平台是否上报该字段以及协议字段映射是否覆盖"),
"field_status_action": stringSchema("实时字段中没有疑似里程字段，优先向源平台核对是否上报总里程"),
},
},
"DataSourceRow": map[string]any{
"type": "object",
"properties": map[string]any{
"id": integerSchema(3),
"protocol": stringSchema("JT808"),
"source_ip": stringSchema("115.231.168.135"),
"latest_source_endpoint": stringSchema("115.231.168.135:41561"),
"platform_name": stringSchema("G7 平台"),
"source_code": stringSchema("G7S"),
"source_kind": stringSchema("PLATFORM"),
"trust_priority": integerSchema(10),
"enabled": map[string]any{"type": "boolean", "example": true},
"first_seen_at": stringSchema("2026-07-09 14:48:30"),
"latest_seen_at": stringSchema("2026-07-12 01:16:04"),
"remark": stringSchema("可信来源"),
"updated_at": stringSchema("2026-07-12 01:16:05"),
},
},
"DataSourceDiagnosticRow": map[string]any{
"type": "object",
"properties": map[string]any{
"id": integerSchema(242190),
"protocol": stringSchema("JT808"),
"source_ip": stringSchema("117.132.194.31"),
"latest_source_endpoint": stringSchema("117.132.194.31:20471"),
"platform_name": stringSchema("G7 平台"),
"source_code": stringSchema("G7S"),
"source_kind": stringSchema("PLATFORM"),
"first_seen_at": stringSchema("2026-07-12 01:00:00"),
"latest_seen_at": stringSchema("2026-07-12 01:36:26"),
"active_span_seconds": integerSchema(2186),
"latest_seen_age_seconds": integerSchema(1800),
"registration_rows": integerSchema(5),
"phone_count": integerSchema(4),
"unknown_vin_rows": integerSchema(2),
"identifier_matched_phones": integerSchema(3),
"unmapped_phone_count": integerSchema(1),
"identifier_match_ratio": map[string]any{"type": "number", "example": 0.75},
"configured_source_code_matched_phones": integerSchema(3),
"configured_source_code_platform_name": stringSchema("东方北斗"),
"configured_source_code_conflict": map[string]any{"type": "boolean", "example": false},
"source_platform_name_mismatch": map[string]any{"type": "boolean", "example": true},
"matched_source_code_count": integerSchema(1),
"candidate_source_code": stringSchema("g7s"),
"candidate_platform_name": stringSchema("G7s"),
"matched_source_codes": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"g7s"}},
"matched_platform_names": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"G7s"}},
"sample_phones": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"13307795425"}},
"reason": stringSchema("candidate_available"),
"recommended_operator_action": stringSchema("可用候选 source_code，执行 identity-import -sync-data-sources -apply 或在来源管理中确认"),
"suggested_source_kind": stringSchema("PLATFORM"),
"suggestion_confidence": stringSchema("HIGH"),
"suggestion_reason": stringSchema("single_source_code_with_many_phones_or_long_activity"),
},
},
"JT808IdentityGapRow": map[string]any{
"type": "object",
"properties": map[string]any{
"phone": stringSchema("13307795425"),
"device_id": stringSchema("TERM-001"),
"plate": stringSchema("沪A63305F"),
"vin": stringSchema("unknown"),
"source_ip": stringSchema("117.132.194.31"),
"source_endpoint": stringSchema("117.132.194.31:20471"),
"source_code": stringSchema("guangan_beidou"),
"platform_name": stringSchema("广安北斗"),
"source_kind": stringSchema("PLATFORM"),
"first_registered_at": stringSchema("2026-07-12 09:00:00"),
"latest_registered_at": stringSchema("2026-07-12 09:00:00"),
"latest_authenticated_at": stringSchema("2026-07-12 09:00:03"),
"latest_seen_at": stringSchema("2026-07-12 22:24:53"),
"latest_seen_age_seconds": integerSchema(32),
"reason": stringSchema("missing_phone_and_plate_binding"),
"recommended_operator_action": stringSchema("将该 phone 或 plate 维护到 vehicle_identifier，并确认 VIN、source_code、oem"),
"raw_frame_query_path": stringSchema("/api/history/raw-frames?protocol=JT808&phone=13307795425&includeFields=true"),
"data_source_query_path": stringSchema("/api/stats/data-sources?protocol=JT808&sourceIP=117.132.194.31"),
},
},
"JT808MappingGapRow": map[string]any{
"type": "object",
"properties": map[string]any{
"phone": stringSchema("64646848246"),
"device_id": stringSchema("TERM-001"),
"plate": stringSchema("粤AG18312"),
"vin": stringSchema("LKLG7C4E8NA774778"),
"source_ip": stringSchema("115.159.85.149"),
"source_endpoint": stringSchema("115.159.85.149:16885"),
"source_code": stringSchema("dongfang_beidou"),
"platform_name": stringSchema("G7易流"),
"source_kind": stringSchema("PLATFORM"),
"identifier_vin": stringSchema(""),
"identifier_plate": stringSchema(""),
"matched_source_codes": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"g7s"}},
"matched_platform_names": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "example": []string{"G7s"}},
"latest_seen_at": stringSchema("2026-07-12 23:38:22"),
"latest_seen_age_seconds": integerSchema(32),
"reason": stringSchema("missing_source_phone_identifier"),
"recommended_action": stringSchema("按当前来源 source_code 维护 JT808_PHONE 到 VIN 的映射；如平台名与 source_code 不一致，先修正来源配置"),
"suggested_source_code": stringSchema("dongfang_beidou"),
"suggested_platform_name": stringSchema("G7易流"),
"suggested_identifier_type": stringSchema("JT808_PHONE"),
"suggested_identifier_value": stringSchema("64646848246"),
"suggested_vin": stringSchema("LKLG7C4E8NA774778"),
"suggested_plate": stringSchema("粤AG18312"),
"raw_frame_query_path": stringSchema("/api/history/raw-frames?protocol=JT808&phone=64646848246&includeFields=true"),
"data_source_query_path": stringSchema("/api/stats/data-sources?protocol=JT808&sourceIP=115.159.85.149"),
"vehicle_identifier_example": stringSchema("protocol=JT808, source_code=dongfang_beidou, identifier_type=JT808_PHONE, identifier_value=64646848246, vin=LKLG7C4E8NA774778, plate=粤AG18312, oem=G7易流"),
},
},
"DataSourceUpdate": map[string]any{
"type": "object",
"properties": map[string]any{
"platform_name": stringSchema("G7 平台"),
"source_code": stringSchema("G7S"),
"source_kind": stringSchema("PLATFORM"),
"trust_priority": integerSchema(10),
"enabled": map[string]any{"type": "boolean", "example": true},
"remark": stringSchema("可信来源"),
},
},
},
@@ -122,6 +738,92 @@ func realtimeOpenAPISpec() map[string]any {
}
}
func dailyMetricSourceParameters() []map[string]any {
parameters := dailyMetricParameters()
parameters = append(parameters,
map[string]any{"name": "sourceIP", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
map[string]any{"name": "qualityStatus", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "NO_PREVIOUS_BASELINE", "INVALID_DELTA"}}, "required": false},
map[string]any{"name": "qualityReason", "in": "query", "schema": map[string]any{"type": "string", "example": "historical_source_baseline"}, "required": false},
map[string]any{"name": "selected", "in": "query", "schema": map[string]any{"type": "boolean"}, "required": false},
)
return parameters
}
func dailyMetricDiagnosisParameters() []map[string]any {
return []map[string]any{
{"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},
{"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"},
{"name": "diagnosis", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "MISSING_DAILY", "NO_SOURCE_SAMPLE", "NO_TOTAL_MILEAGE"}}, "required": false},
{"name": "reason", "in": "query", "schema": map[string]any{"type": "string", "enum": dailyMetricDiagnosisReasonEnum()}, "required": false},
{"name": "severity", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"ok", "pipeline", "source_data"}}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false, "description": "默认 false，不执行 COUNT(*)；true 时返回精确总数。"},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
}
}
func dailyMetricDiagnosisSummaryParameters() []map[string]any {
return []map[string]any{
{"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},
{"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"},
}
}
func dailyMetricDiagnosisReasonSummaryParameters() []map[string]any {
return []map[string]any{
{"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},
{"name": "date", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false, "description": "按东八区业务日期诊断；不传时默认服务当前日期。"},
{"name": "diagnosis", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"OK", "MISSING_DAILY", "NO_SOURCE_SAMPLE", "NO_TOTAL_MILEAGE"}}, "required": false},
{"name": "reason", "in": "query", "schema": map[string]any{"type": "string", "enum": dailyMetricDiagnosisReasonEnum()}, "required": false},
{"name": "severity", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"ok", "pipeline", "source_data"}}, "required": false},
}
}
func dailyMetricDiagnosisReasonEnum() []string {
return []string{
"daily_metric_exists",
"source_samples_all_invalid",
"source_samples_all_excluded",
"source_samples_exist_but_final_metric_missing",
"realtime_location_has_total_mileage_but_no_stat_sample",
"realtime_total_mileage_missing",
"realtime_total_mileage_non_positive",
"realtime_total_mileage_time_missing",
"realtime_total_mileage_not_reported_on_stat_date",
"realtime_active_without_total_mileage",
}
}
func dailyMetricMileageFieldStatusEnum() []string {
return []string{
"daily_metric_exists",
"source_sample_exists",
"standard_field_not_consumed",
"standard_field_non_positive",
"standard_field_time_missing",
"standard_field_stale",
"candidate_field_unmapped",
"mapped_protocol_field_without_fresh_evidence",
"no_candidate_mileage_field",
"no_realtime_fields",
}
}
func dailyMetricParameters() []map[string]any {
return []map[string]any{
{"name": "vin", "in": "query", "schema": map[string]any{"type": "string"}, "required": false},
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},
{"name": "dateFrom", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false},
{"name": "dateTo", "in": "query", "schema": map[string]any{"type": "string", "format": "date"}, "required": false},
{"name": "includeTotal", "in": "query", "schema": map[string]any{"type": "boolean", "default": false}, "required": false, "description": "默认 false，不执行 COUNT(*)；true 时返回精确总数。"},
{"name": "limit", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 1, "maximum": 1000, "default": 50}, "required": false},
{"name": "offset", "in": "query", "schema": map[string]any{"type": "integer", "minimum": 0, "maximum": 1000000, "default": 0}, "required": false},
}
}
func commonRealtimeParameters() []map[string]any {
return []map[string]any{
{"name": "protocol", "in": "query", "schema": map[string]any{"type": "string", "enum": []string{"GB32960", "JT808", "YUTONG_MQTT"}}, "required": false},

33
go/vehicle-gateway/internal/realtime/openapi_test.go
View File

@@ -17,17 +17,46 @@ func TestOpenAPIHandlerDocumentsRealtimeSnapshotAndLocation(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"/api/realtime/snapshots"`, `"/api/realtime/locations"`, `"vehicle_realtime_snapshot"`, `"vehicle_realtime_location"`} {
for _, want := range []string{
`"/api/realtime/snapshots"`,
`"/api/realtime/locations"`,
`"/api/stats/daily-metrics"`,
`"/api/stats/daily-metrics/sources"`,
`"/api/stats/daily-metrics/sources/quality"`,
`"/api/stats/daily-metrics/sources/selection"`,
`"/api/stats/daily-metrics/diagnostics"`,
`"/api/stats/daily-metrics/diagnostics/summary"`,
`"/api/stats/daily-metrics/diagnostics/reasons"`,
`"/api/stats/daily-metrics/diagnostics/field-status"`,
`"/api/stats/data-sources"`,
`"/api/stats/data-sources/diagnostics"`,
`"/api/stats/data-sources/kind-suggestions"`,
`"/api/stats/data-sources/jt808-identity-gaps"`,
`"/api/stats/data-sources/jt808-mapping-gaps"`,
`"/api/stats/data-sources/{id}"`,
`"vehicle_realtime_snapshot"`,
`"vehicle_realtime_location"`,
`vehicle_daily_mileage`,
`vehicle_daily_mileage_source`,
`"vehicle_data_source"`,
`"Stats API"`,
`"Data Source API"`,
} {
if !strings.Contains(body, want) {
t.Fatalf("openapi missing %s: %s", want, body)
}
}
for _, want := range []string{`"daily_mileage_km"`, `"latest_total_mileage_km"`, `"source_key"`, `"source_count"`, `"vehicle_count"`, `"selected_count"`, `"sample_count"`, `"quality_status"`, `"quality_reason"`, `"is_selected"`, `"selection_status"`, `"selection_reason"`, `"selection_action"`, `"lower_trust_priority_or_sample_count"`, `"platform_name"`, `"source_code"`, `"source_kind"`, `"sourceKind"`, `"trust_priority"`, `"enabled"`, `"latest_seen_at"`, `"candidate_source_code"`, `"recommended_operator_action"`, `"suggested_source_kind"`, `"suggestion_confidence"`, `"NO_SOURCE_SAMPLE"`, `"NO_TOTAL_MILEAGE"`, `"source_sample_count"`, `"realtime_total_mileage_km"`, `"total_mileage_event_time"`, `"realtime_total_mileage_event_time"`, `"active_count"`, `"actionable_issue_count"`, `"vehicle_total"`, `"pipeline_issue_total"`, `"source_data_issue_total"`, `"severity"`, `"field_status_severity"`, `"realtime_total_mileage_not_reported_on_stat_date"`, `"realtime_mileage_field_status"`, `"field_status_action"`, `"candidate_field_unmapped"`, `"standard_field_stale"`, `"missing_phone_and_plate_binding"`, `"missing_source_phone_identifier"`, `"configured_source_code_platform_name"`, `"source_platform_name_mismatch"`, `"suggested_identifier_value"`, `"vehicle_identifier_example"`, `"raw_frame_query_path"`, `"data_source_query_path"`} {
if !strings.Contains(body, want) {
t.Fatalf("openapi should document data source field %s: %s", want, body)
}
}
for _, removed := range []string{`"/api/realtime/kv"`, `"vehicle_realtime_kv"`, `"Realtime KV API"`} {
if strings.Contains(body, removed) {
t.Fatalf("openapi should not expose removed MySQL realtime kv API %s: %s", removed, body)
}
}
for _, want := range []string{`"includeTotal"`, `默认不执行 COUNT(*)`, `total 默认表示本页返回条数`} {
for _, want := range []string{`"includeTotal"`, `"sourceCodeMissing"`, `默认不执行 COUNT(*)`, `total 默认表示本页返回条数`} {
if !strings.Contains(body, want) {
t.Fatalf("openapi should document lightweight total semantics, missing %s: %s", want, body)
}

331
go/vehicle-gateway/internal/realtime/repository.go
View File

@@ -20,6 +20,32 @@ type Repository struct {
cfg Config
}
type FastUpdateResult struct {
EnvelopesSeen int
EnvelopesUpdated int
EnvelopesSkippedNonRealtime int
EnvelopesSkippedMissingVIN int
EnvelopesSkippedMissingVehicleKey int
EnvelopesSkippedMissingFields int
FieldsSeen int
FieldsWritten int
FieldsSkippedStale int
}
func (r FastUpdateResult) Add(other FastUpdateResult) FastUpdateResult {
return FastUpdateResult{
EnvelopesSeen: r.EnvelopesSeen + other.EnvelopesSeen,
EnvelopesUpdated: r.EnvelopesUpdated + other.EnvelopesUpdated,
EnvelopesSkippedNonRealtime: r.EnvelopesSkippedNonRealtime + other.EnvelopesSkippedNonRealtime,
EnvelopesSkippedMissingVIN: r.EnvelopesSkippedMissingVIN + other.EnvelopesSkippedMissingVIN,
EnvelopesSkippedMissingVehicleKey: r.EnvelopesSkippedMissingVehicleKey + other.EnvelopesSkippedMissingVehicleKey,
EnvelopesSkippedMissingFields: r.EnvelopesSkippedMissingFields + other.EnvelopesSkippedMissingFields,
FieldsSeen: r.FieldsSeen + other.FieldsSeen,
FieldsWritten: r.FieldsWritten + other.FieldsWritten,
FieldsSkippedStale: r.FieldsSkippedStale + other.FieldsSkippedStale,
}
}
func NewRepository(client *redis.Client, cfg Config) *Repository {
if client == nil {
panic("redis client must not be nil")
@@ -28,48 +54,82 @@ func NewRepository(client *redis.Client, cfg Config) *Repository {
}
func (r *Repository) FastUpdate(ctx context.Context, env envelope.FrameEnvelope) error {
_, err := r.FastUpdateWithResult(ctx, env)
return err
}
func (r *Repository) FastUpdateWithResult(ctx context.Context, env envelope.FrameEnvelope) (FastUpdateResult, error) {
result := FastUpdateResult{EnvelopesSeen: 1}
if !envelope.IsRealtimeTelemetryFrame(env) {
return nil
result.EnvelopesSkippedNonRealtime = 1
return result, nil
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
return nil
result.EnvelopesSkippedMissingVIN = 1
return result, nil
}
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
return nil
result.EnvelopesSkippedMissingVehicleKey = 1
return result, nil
}
if len(env.ParsedFields) == 0 {
result.EnvelopesSkippedMissingFields = 1
return result, nil
}
return r.setFastProjection(ctx, vehicleKey, vin, env)
}
func (r *Repository) FastUpdateBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
_, err := r.FastUpdateBatchWithResult(ctx, envs)
return err
}
func (r *Repository) FastUpdateBatchWithResult(ctx context.Context, envs []envelope.FrameEnvelope) (FastUpdateResult, error) {
if len(envs) == 0 {
return nil
return FastUpdateResult{}, nil
}
pipe := r.client.Pipeline()
queued := 0
var result FastUpdateResult
queued := make([]queuedFastProjection, 0, len(envs))
for _, env := range envs {
result.EnvelopesSeen++
if !envelope.IsRealtimeTelemetryFrame(env) {
result.EnvelopesSkippedNonRealtime++
continue
}
vin := strings.TrimSpace(env.VIN)
if vin == "" {
result.EnvelopesSkippedMissingVIN++
continue
}
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
result.EnvelopesSkippedMissingVehicleKey++
continue
}
if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil {
return err
if len(env.ParsedFields) == 0 {
result.EnvelopesSkippedMissingFields++
continue
}
queued++
queuedProjection, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env)
if err != nil {
return FastUpdateResult{}, err
}
queued = append(queued, queuedProjection)
}
if queued == 0 {
return nil
if len(queued) == 0 {
return result, nil
}
_, err := pipe.Exec(ctx)
return err
if err != nil {
return FastUpdateResult{}, err
}
for _, item := range queued {
result = result.Add(item.result())
}
return result, nil
}
func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -85,10 +145,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
return nil
}
nowMS := time.Now().UnixMilli()
eventMS := env.EventTimeMS
if eventMS <= 0 {
eventMS = env.ReceivedAtMS
}
eventMS, _ := envelope.NormalizedEventTimeMS(env)
protocolSnapshot := Snapshot{
VehicleKey: vehicleKey,
VIN: vin,
@@ -129,7 +186,7 @@ func (r *Repository) Update(ctx context.Context, env envelope.FrameEnvelope) err
if err := r.setJSON(ctx, realtimeRawKey(vehicleKey, env.Protocol), protocolSnapshot.Parsed, r.cfg.ttl()); err != nil {
return err
}
if err := r.setKV(ctx, vin, env, protocolSnapshot.Parsed); err != nil {
if err := r.setKV(ctx, vin, env); err != nil {
return err
}
@@ -377,10 +434,7 @@ func (r *Repository) setJSON(ctx context.Context, key string, value any, ttl tim
return r.client.Set(ctx, key, payload, ttl).Err()
}
func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope, parsed map[string]any) error {
if len(env.ParsedFields) == 0 && len(parsed) > 0 {
env.Parsed = parsed
}
func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEnvelope) error {
values, types := realtimeKVMapsForEnvelope(env)
if len(values) == 0 {
return nil
@@ -394,24 +448,43 @@ func (r *Repository) setKV(ctx context.Context, vin string, env envelope.FrameEn
"source_endpoint": env.SourceEndpoint,
"field_mapping": realtimeFieldMappingVersion,
}
pipe := r.client.Pipeline()
pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values)
pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types)
pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta)
_, err := pipe.Exec(ctx)
return err
return evalGuardedRealtimeKV(ctx, r.client, env.Protocol, vin, eventTimeOrReceivedMS(env), values, types, meta).Err()
}
func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) error {
func (r *Repository) setFastProjection(ctx context.Context, vehicleKey string, vin string, env envelope.FrameEnvelope) (FastUpdateResult, error) {
pipe := r.client.Pipeline()
if err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env); err != nil {
return err
queued, err := r.queueFastProjection(ctx, pipe, vehicleKey, vin, env)
if err != nil {
return FastUpdateResult{}, err
}
_, err := pipe.Exec(ctx)
return err
if _, err := pipe.Exec(ctx); err != nil {
return FastUpdateResult{}, err
}
result := queued.result()
result.EnvelopesSeen = 1
return result, nil
}
func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) error {
type queuedFastProjection struct {
fieldsSeen int
writeCmd *redis.Cmd
}
func (q queuedFastProjection) result() FastUpdateResult {
written := redisCmdInt(q.writeCmd)
skipped := q.fieldsSeen - written
if skipped < 0 {
skipped = 0
}
return FastUpdateResult{
EnvelopesUpdated: 1,
FieldsSeen: q.fieldsSeen,
FieldsWritten: written,
FieldsSkippedStale: skipped,
}
}
func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipeliner, vehicleKey string, vin string, env envelope.FrameEnvelope) (queuedFastProjection, error) {
values, types := realtimeKVMapsForEnvelope(env)
eventTimeMS := eventTimeOrReceivedMS(env)
offlineAfterMS := env.ReceivedAtMS + r.cfg.ttl().Milliseconds()
@@ -428,7 +501,7 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin
}
payload, err := json.Marshal(online)
if err != nil {
return err
return queuedFastProjection{}, err
}
meta := map[string]any{
"event_time_ms": strconv.FormatInt(eventTimeMS, 10),
@@ -449,16 +522,171 @@ func (r *Repository) queueFastProjection(ctx context.Context, pipe redis.Pipelin
"ttl_seconds": strconv.FormatInt(int64(r.cfg.ttl().Seconds()), 10),
"source_endpoint": env.SourceEndpoint,
}
queued := queuedFastProjection{fieldsSeen: len(values)}
if len(values) > 0 {
pipe.HSet(ctx, realtimeKVValuesKey(env.Protocol, vin), values)
pipe.HSet(ctx, realtimeKVTypesKey(env.Protocol, vin), types)
pipe.HSet(ctx, realtimeKVMetaKey(env.Protocol, vin), meta)
queued.writeCmd = evalGuardedRealtimeKV(ctx, pipe, env.Protocol, vin, eventTimeMS, values, types, meta)
}
pipe.Set(ctx, onlineKey(env.Protocol, vin), payload, r.cfg.ttl())
pipe.HSet(ctx, onlineStateKey(env.Protocol, vin), state)
pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(env.ReceivedAtMS), Member: onlineMember(env.Protocol, vin)})
pipe.SAdd(ctx, realtimeIndexKey(env.Protocol), vin)
return nil
if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil {
return queuedFastProjection{}, err
}
return queued, nil
}
const guardedRealtimeKVScript = `
local incoming = tonumber(ARGV[1]) or 0
local value_count = tonumber(ARGV[2]) or 0
local idx = 3
local written = 0
for i = 1, value_count do
local field = ARGV[idx]
local value = ARGV[idx + 1]
idx = idx + 2
local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0
if current <= incoming then
redis.call('HSET', KEYS[1], field, value)
redis.call('HSET', KEYS[3], field, incoming)
written = written + 1
end
end
local type_count = tonumber(ARGV[idx]) or 0
idx = idx + 1
for i = 1, type_count do
local field = ARGV[idx]
local value_type = ARGV[idx + 1]
idx = idx + 2
local current = tonumber(redis.call('HGET', KEYS[3], field) or '') or 0
if current <= incoming then
redis.call('HSET', KEYS[2], field, value_type)
end
end
local meta_count = tonumber(ARGV[idx]) or 0
idx = idx + 1
local current_meta = tonumber(redis.call('HGET', KEYS[4], 'event_time_ms') or '') or 0
if current_meta <= incoming then
for i = 1, meta_count do
redis.call('HSET', KEYS[4], ARGV[idx], ARGV[idx + 1])
idx = idx + 2
end
end
return written
`
const guardedOnlineStatusScript = `
local incoming = tonumber(ARGV[1]) or 0
local ttl_ms = tonumber(ARGV[2]) or 60000
local payload = ARGV[3]
local member = ARGV[4]
local vin = ARGV[5]
local state_count = tonumber(ARGV[6]) or 0
local idx = 7
local current = tonumber(redis.call('HGET', KEYS[2], 'last_seen_ms') or '') or 0
redis.call('SADD', KEYS[4], vin)
if current <= incoming then
redis.call('SET', KEYS[1], payload, 'PX', ttl_ms)
for i = 1, state_count do
redis.call('HSET', KEYS[2], ARGV[idx], ARGV[idx + 1])
idx = idx + 2
end
redis.call('ZADD', KEYS[3], incoming, member)
return 1
end
return 0
`
type redisEvaler interface {
Eval(ctx context.Context, script string, keys []string, args ...any) *redis.Cmd
}
func redisCmdInt(cmd *redis.Cmd) int {
if cmd == nil {
return 0
}
value, err := cmd.Int64()
if err != nil {
return 0
}
return int(value)
}
func evalGuardedRealtimeKV(ctx context.Context, evaler redisEvaler, protocol envelope.Protocol, vin string, eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) *redis.Cmd {
keys := []string{
realtimeKVValuesKey(protocol, vin),
realtimeKVTypesKey(protocol, vin),
realtimeKVTimesKey(protocol, vin),
realtimeKVMetaKey(protocol, vin),
}
args := guardedRealtimeKVArgs(eventTimeMS, values, types, meta)
return evaler.Eval(ctx, guardedRealtimeKVScript, keys, args...)
}
func evalGuardedOnlineStatus(ctx context.Context, evaler redisEvaler, online OnlineStatus, state map[string]any, payload []byte, ttl time.Duration) (*redis.Cmd, error) {
protocol := online.Protocol
if protocol == "" && len(online.Protocols) > 0 {
protocol = online.Protocols[0]
}
vin := strings.TrimSpace(online.VIN)
if protocol == "" || vin == "" {
return nil, nil
}
if len(payload) == 0 {
var err error
payload, err = json.Marshal(online)
if err != nil {
return nil, err
}
}
if ttl <= 0 {
ttl = time.Minute
}
keys := []string{
onlineKey(protocol, vin),
onlineStateKey(protocol, vin),
"vehicle:last_seen",
realtimeIndexKey(protocol),
}
args := guardedOnlineStatusArgs(online, protocol, state, payload, ttl)
return evaler.Eval(ctx, guardedOnlineStatusScript, keys, args...), nil
}
func guardedOnlineStatusArgs(online OnlineStatus, protocol envelope.Protocol, state map[string]any, payload []byte, ttl time.Duration) []any {
args := []any{
strconv.FormatInt(online.LastSeenMS, 10),
strconv.FormatInt(ttl.Milliseconds(), 10),
string(payload),
onlineMember(protocol, online.VIN),
strings.TrimSpace(online.VIN),
}
return appendMapPairs(args, state)
}
func guardedRealtimeKVArgs(eventTimeMS int64, values map[string]any, types map[string]any, meta map[string]any) []any {
args := []any{strconv.FormatInt(eventTimeMS, 10)}
args = appendMapPairs(args, values)
args = appendMapPairs(args, types)
args = appendMapPairs(args, meta)
return args
}
func appendMapPairs(args []any, values map[string]any) []any {
keys := make([]string, 0, len(values))
for key := range values {
key = strings.TrimSpace(key)
if key == "" {
continue
}
keys = append(keys, key)
}
sort.Strings(keys)
args = append(args, strconv.Itoa(len(keys)))
for _, key := range keys {
args = append(args, key, fmt.Sprint(values[key]))
}
return args
}
func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[string]any) {
@@ -469,6 +697,9 @@ func realtimeKVMapsForEnvelope(env envelope.FrameEnvelope) (map[string]any, map[
if strings.TrimSpace(field) == "" {
continue
}
if isRealtimeTotalMileageField(field) && !positiveNumber(value) {
continue
}
stringValue, valueType, ok := stringifyKVValue(value)
if !ok {
continue
@@ -528,7 +759,6 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e
if online.OfflineAfterMS <= 0 {
online.OfflineAfterMS = online.LastSeenMS + r.cfg.ttl().Milliseconds()
}
member := onlineMember(protocol, online.VIN)
state := map[string]any{
"vehicle_key": online.VehicleKey,
"vin": online.VIN,
@@ -544,10 +774,9 @@ func (r *Repository) setOnlineStatus(ctx context.Context, online OnlineStatus) e
if err != nil {
return err
}
pipe.Set(ctx, onlineKey(protocol, online.VIN), payload, r.cfg.ttl())
pipe.HSet(ctx, onlineStateKey(protocol, online.VIN), state)
pipe.ZAdd(ctx, "vehicle:last_seen", redis.Z{Score: float64(online.LastSeenMS), Member: member})
pipe.SAdd(ctx, realtimeIndexKey(protocol), online.VIN)
if _, err := evalGuardedOnlineStatus(ctx, pipe, online, state, payload, r.cfg.ttl()); err != nil {
return err
}
_, err = pipe.Exec(ctx)
return err
}
@@ -879,6 +1108,10 @@ func realtimeKVTypesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":types"
}
func realtimeKVTimesKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":times"
}
func realtimeKVMetaKey(protocol envelope.Protocol, vin string) string {
return "vehicle:rt-kv:" + string(protocol) + ":" + strings.TrimSpace(vin) + ":meta"
}
@@ -897,10 +1130,8 @@ func realtimeKVFieldPath(domain string, field string) string {
}
func eventTimeOrReceivedMS(env envelope.FrameEnvelope) int64 {
if env.EventTimeMS > 0 {
return env.EventTimeMS
}
return env.ReceivedAtMS
eventMS, _ := envelope.NormalizedEventTimeMS(env)
return eventMS
}
func onlineKey(protocol envelope.Protocol, vin string) string {

339
go/vehicle-gateway/internal/realtime/repository_test.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
@@ -133,6 +134,50 @@ func TestRepositoryStoresFullParsedOnlyInRealtimeRawAndMergesGB32960Units(t *tes
}
}
func TestRepositoryNormalizesFarFutureEventTime(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC).UnixMilli()
futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC).UnixMilli()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: futureEvent,
ReceivedAtMS: received,
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1}},
ParsedFields: map[string]any{
"jt808.location.longitude": 121.1,
"jt808.location.latitude": 30.1,
"jt808.location.total_mileage_km": 10241.2,
},
Fields: map[string]any{
envelope.FieldLongitude: 121.1,
envelope.FieldLatitude: 30.1,
envelope.FieldTotalMileageKM: 10241.2,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
protocol, err := repo.GetProtocol(ctx, "VIN001", envelope.ProtocolJT808)
if err != nil {
t.Fatalf("GetProtocol() error = %v", err)
}
if protocol.EventTimeMS != received {
t.Fatalf("protocol event time = %d, want received %d", protocol.EventTimeMS, received)
}
meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("HGetAll meta error = %v", err)
}
if meta["event_time_ms"] != strconv.FormatInt(received, 10) {
t.Fatalf("rt-kv event_time_ms = %q, want received %d", meta["event_time_ms"], received)
}
}
func TestRepositoryMergesNestedGB32960MotorSlicesBySerialNo(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
@@ -337,6 +382,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) {
}},
},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
"gb32960.gd_fc_stack.type": "0x30",
"gb32960.gd_fc_stack.stack_count": 1,
"gb32960.gd_fc_stack.stack_water_outlet_temp_c": 63,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
@@ -358,6 +410,13 @@ func TestRepositoryWritesRealtimeKVHashes(t *testing.T) {
if typeKV["gb32960.vehicle.soc_percent"] != "number" || typeKV["gb32960.gd_fc_stack.type"] != "string" {
t.Fatalf("types kv = %#v", typeKV)
}
timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result()
if err != nil {
t.Fatalf("times kv HGetAll error = %v", err)
}
if timeKV["gb32960.vehicle.soc_percent"] != "1000" || timeKV["gb32960.gd_fc_stack.stack_water_outlet_temp_c"] != "1000" {
t.Fatalf("times kv = %#v", timeKV)
}
metaKV, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:meta").Result()
if err != nil {
t.Fatalf("meta kv HGetAll error = %v", err)
@@ -388,6 +447,10 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}},
},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
}
@@ -406,9 +469,19 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
if typeKV["gb32960.vehicle.soc_percent"] != "number" {
t.Fatalf("types kv = %#v", typeKV)
}
timeKV, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Result()
if err != nil {
t.Fatalf("times kv HGetAll error = %v", err)
}
if timeKV["gb32960.vehicle.soc_percent"] != "1000" {
t.Fatalf("times kv = %#v", timeKV)
}
if ttl := repo.client.TTL(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val(); ttl != -1 {
t.Fatalf("kv ttl should not expire, got %v", ttl)
}
if ttl := repo.client.TTL(ctx, realtimeKVTimesKey(envelope.ProtocolGB32960, "VIN001")).Val(); ttl != -1 {
t.Fatalf("kv times ttl should not expire, got %v", ttl)
}
if ttl := repo.client.TTL(ctx, "vehicle:online:GB32960:VIN001").Val(); ttl <= 0 || ttl > time.Minute {
t.Fatalf("online ttl = %v, want within 1 minute", ttl)
}
@@ -430,12 +503,138 @@ func TestRepositoryFastUpdateOnlyWritesPermanentKVAndMinuteOnline(t *testing.T)
}
}
func TestRepositoryFastUpdateDoesNotLetOlderFramesOverwriteNewerFields(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
firstResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: 2000,
ReceivedAtMS: 2100,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 50,
},
})
if err != nil {
t.Fatalf("newer FastUpdate() error = %v", err)
}
if firstResult.FieldsSeen != 1 || firstResult.FieldsWritten != 1 || firstResult.FieldsSkippedStale != 0 {
t.Fatalf("newer result = %#v, want 1/1/0", firstResult)
}
if firstResult.EnvelopesSeen != 1 || firstResult.EnvelopesUpdated != 1 {
t.Fatalf("newer envelope result = %#v, want seen=1 updated=1", firstResult)
}
secondResult, err := repo.FastUpdateWithResult(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 2200,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 10,
"jt808.location.latitude": 30.5,
},
})
if err != nil {
t.Fatalf("older FastUpdate() error = %v", err)
}
if secondResult.FieldsSeen != 2 || secondResult.FieldsWritten != 1 || secondResult.FieldsSkippedStale != 1 {
t.Fatalf("older result = %#v, want 2/1/1", secondResult)
}
if secondResult.EnvelopesSeen != 1 || secondResult.EnvelopesUpdated != 1 {
t.Fatalf("older envelope result = %#v, want seen=1 updated=1", secondResult)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("values HGetAll error = %v", err)
}
if values["jt808.location.speed_kmh"] != "50" {
t.Fatalf("older frame overwrote speed: %#v", values)
}
if values["jt808.location.latitude"] != "30.5" {
t.Fatalf("older frame should still add previously unseen latitude: %#v", values)
}
times, err := repo.client.HGetAll(ctx, realtimeKVTimesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("times HGetAll error = %v", err)
}
if times["jt808.location.speed_kmh"] != "2000" || times["jt808.location.latitude"] != "1000" {
t.Fatalf("field times = %#v", times)
}
meta, err := repo.client.HGetAll(ctx, realtimeKVMetaKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("meta HGetAll error = %v", err)
}
if meta["event_time_ms"] != "2000" {
t.Fatalf("meta should keep latest frame time: %#v", meta)
}
}
func TestRepositoryFastUpdateDoesNotLetOlderReceivedFrameOverwriteOnlineState(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
SourceEndpoint: "newer.example:808",
EventTimeMS: 2000,
ReceivedAtMS: 5000,
ParsedFields: map[string]any{
"jt808.location.speed_kmh": 50,
},
}); err != nil {
t.Fatalf("newer FastUpdate() error = %v", err)
}
if err := repo.FastUpdate(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
SourceEndpoint: "older.example:808",
EventTimeMS: 1000,
ReceivedAtMS: 4000,
ParsedFields: map[string]any{
"jt808.location.latitude": 30.5,
},
}); err != nil {
t.Fatalf("older FastUpdate() error = %v", err)
}
state, err := repo.client.HGetAll(ctx, "vehicle:online-state:JT808:VIN001").Result()
if err != nil {
t.Fatalf("online state HGetAll error = %v", err)
}
if state["last_seen_ms"] != "5000" || state["offline_after_ms"] != "65000" || state["source_endpoint"] != "newer.example:808" {
t.Fatalf("older received frame should not overwrite online state: %#v", state)
}
score, err := repo.client.ZScore(ctx, "vehicle:last_seen", "JT808:VIN001").Result()
if err != nil {
t.Fatalf("last_seen ZScore error = %v", err)
}
if score != 5000 {
t.Fatalf("last_seen score = %v, want 5000", score)
}
status, err := repo.IsOnline(ctx, "VIN001")
if err != nil {
t.Fatalf("IsOnline() error = %v", err)
}
if status.LastSeenMS != 5000 || status.SourceEndpoint != "newer.example:808" {
t.Fatalf("online status should keep newer received frame: %#v", status)
}
}
func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
err := repo.FastUpdateBatch(ctx, []envelope.FrameEnvelope{
result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{
{
Protocol: envelope.ProtocolGB32960,
VIN: "VIN001",
@@ -444,6 +643,10 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{
"gb32960.vehicle.type": "0x01",
"gb32960.vehicle.soc_percent": 88.0,
},
},
{
Protocol: envelope.ProtocolJT808,
@@ -452,12 +655,17 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
ReceivedAtMS: 2200,
MessageID: "0x0200",
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}},
ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2},
Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2},
},
})
if err != nil {
t.Fatalf("FastUpdateBatch() error = %v", err)
}
if result.FieldsSeen != 4 || result.FieldsWritten != 4 || result.FieldsSkippedStale != 0 ||
result.EnvelopesSeen != 2 || result.EnvelopesUpdated != 2 {
t.Fatalf("FastUpdateBatchWithResult() result = %#v, want fields 4/4/0 and envelopes 2/2", result)
}
gbValues, err := repo.client.HGetAll(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Result()
if err != nil {
@@ -481,6 +689,55 @@ func TestRepositoryFastUpdateBatchWritesMultipleRealtimeProjections(t *testing.T
}
}
func TestRepositoryFastUpdateReportsSkippedEnvelopesWithoutWritingRedis(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
result, err := repo.FastUpdateBatchWithResult(ctx, []envelope.FrameEnvelope{
{
Protocol: envelope.ProtocolGB32960,
MessageID: "0x01",
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1000,
Parsed: map[string]any{"platform_login": map[string]any{"username": "Hyundai"}},
},
{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
EventTimeMS: 2000,
ReceivedAtMS: 2000,
ParsedFields: map[string]any{"jt808.location.speed_kmh": 30},
},
{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN002",
EventTimeMS: 3000,
ReceivedAtMS: 3000,
Parsed: map[string]any{"location": map[string]any{"speed_kmh": 30}},
},
})
if err != nil {
t.Fatalf("FastUpdateBatchWithResult() error = %v", err)
}
if result.EnvelopesSeen != 3 || result.EnvelopesUpdated != 0 ||
result.EnvelopesSkippedNonRealtime != 1 || result.EnvelopesSkippedMissingVIN != 1 ||
result.EnvelopesSkippedMissingFields != 1 {
t.Fatalf("result = %#v, want non-realtime, missing vin and missing parsed fields skips", result)
}
if result.FieldsSeen != 0 || result.FieldsWritten != 0 || result.FieldsSkippedStale != 0 {
t.Fatalf("field result = %#v, want no field writes", result)
}
if repo.client.Exists(ctx, "vehicle:rt-kv:GB32960:VIN001:values").Val() != 0 {
t.Fatal("non-realtime frame should not write realtime kv")
}
if repo.client.Exists(ctx, "vehicle:online:JT808:VIN002").Val() != 0 {
t.Fatal("missing parsed fields frame should not mark vehicle online")
}
}
func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
@@ -495,6 +752,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0},
},
{
Protocol: envelope.ProtocolJT808,
@@ -503,6 +761,7 @@ func TestRepositoryListsOnlineStatusesAndPipelineSummary(t *testing.T) {
ReceivedAtMS: 2200,
MessageID: "0x0200",
Parsed: map[string]any{"location": map[string]any{"longitude": 121.1, "latitude": 30.2}},
ParsedFields: map[string]any{"jt808.location.longitude": 121.1, "jt808.location.latitude": 30.2},
Fields: map[string]any{envelope.FieldLongitude: 121.1, envelope.FieldLatitude: 30.2},
},
} {
@@ -548,6 +807,7 @@ func TestOnlineIndexHandlerReturnsPagedOnlineStatuses(t *testing.T) {
Parsed: map[string]any{
"data_units": []any{map[string]any{"type": "0x01", "name": "vehicle", "value": map[string]any{"soc_percent": 88.0}}},
},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 88.0},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
}
@@ -579,6 +839,7 @@ func TestPipelineDebugHandlerReturnsProtocolSummary(t *testing.T) {
EventTimeMS: 3000,
ReceivedAtMS: 3300,
Parsed: map[string]any{"data": map[string]any{"TOTAL_MILEAGE": 1}},
ParsedFields: map[string]any{"yutong_mqtt.data.total_mileage": 1},
Fields: map[string]any{envelope.FieldTotalMileageKM: 1},
}); err != nil {
t.Fatalf("FastUpdate() error = %v", err)
@@ -728,6 +989,82 @@ func TestRepositoryUsesEnvelopeParsedFieldsForRealtimeKV(t *testing.T) {
}
}
func TestRepositoryDropsNonPositiveTotalMileageFromRealtimeKV(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
VIN: "VIN001",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
MessageID: "0x0200",
ParsedFields: map[string]any{
"jt808.location.latitude": "31.259555",
"jt808.location.longitude": "119.892413",
"jt808.location.total_mileage_km": "0",
},
Fields: map[string]any{
envelope.FieldLatitude: 31.259555,
envelope.FieldLongitude: 119.892413,
envelope.FieldTotalMileageKM: 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolJT808, "VIN001")).Result()
if err != nil {
t.Fatalf("HGetAll() error = %v", err)
}
if _, exists := values["jt808.location.total_mileage_km"]; exists {
t.Fatalf("realtime kv should drop non-positive mileage: %#v", values)
}
if values["jt808.location.latitude"] != "31.259555" {
t.Fatalf("realtime kv should keep valid fields: %#v", values)
}
}
func TestRepositoryDropsNonPositiveRawTotalMileageFromRealtimeKV(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()
ctx := context.Background()
if err := repo.Update(ctx, envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
VIN: "LMRKH9AC3R1004101",
EventTimeMS: 1000,
ReceivedAtMS: 1100,
MessageID: "MQTT",
Parsed: map[string]any{
"data": map[string]any{
"LATITUDE": 30.590921,
"LONGITUDE": 121.075044,
"TOTAL_MILEAGE": 0,
},
},
ParsedFields: map[string]any{
"yutong_mqtt.data.latitude": 30.590921,
"yutong_mqtt.data.longitude": 121.075044,
"yutong_mqtt.data.total_mileage": 0,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
values, err := repo.client.HGetAll(ctx, realtimeKVValuesKey(envelope.ProtocolYutongMQTT, "LMRKH9AC3R1004101")).Result()
if err != nil {
t.Fatalf("HGetAll() error = %v", err)
}
if _, exists := values["yutong_mqtt.data.total_mileage"]; exists {
t.Fatalf("realtime kv should drop non-positive raw mileage: %#v", values)
}
if values["yutong_mqtt.data.latitude"] != "30.590921" {
t.Fatalf("realtime kv should keep valid fields: %#v", values)
}
}
func TestRepositorySkipsFramesWithoutVIN(t *testing.T) {
repo, closeFn := newTestRepository(t)
defer closeFn()

393
go/vehicle-gateway/internal/realtime/snapshot_writer.go
View File

@@ -10,6 +10,7 @@ import (
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)
type SnapshotExecer interface {
@@ -21,11 +22,20 @@ type PlateResolver interface {
}
type CachedPlateResolver struct {
delegate PlateResolver
ttl time.Duration
now func() time.Time
mu sync.Mutex
entries map[string]cachedPlateEntry
delegate PlateResolver
ttl time.Duration
maxEntries int
now func() time.Time
observe func(PlateCacheStats)
mu sync.Mutex
entries map[string]cachedPlateEntry
evictions int
}
type PlateCacheStats struct {
Entries int
MaxEntries int
Evictions int
}
type cachedPlateEntry struct {
@@ -35,18 +45,38 @@ type cachedPlateEntry struct {
}
func NewCachedPlateResolver(delegate PlateResolver, ttl time.Duration) *CachedPlateResolver {
return NewCachedPlateResolverWithMaxEntries(delegate, ttl, 200000)
}
func NewCachedPlateResolverWithMaxEntries(delegate PlateResolver, ttl time.Duration, maxEntries int) *CachedPlateResolver {
if delegate == nil {
panic("cached plate resolver delegate must not be nil")
}
if ttl <= 0 {
ttl = 10 * time.Minute
}
return &CachedPlateResolver{
delegate: delegate,
ttl: ttl,
now: time.Now,
entries: map[string]cachedPlateEntry{},
if maxEntries < 0 {
maxEntries = 0
}
return &CachedPlateResolver{
delegate: delegate,
ttl: ttl,
maxEntries: maxEntries,
now: time.Now,
entries: map[string]cachedPlateEntry{},
}
}
func (r *CachedPlateResolver) SetStatsObserver(observe func(PlateCacheStats)) {
r.mu.Lock()
r.observe = observe
r.mu.Unlock()
}
func (r *CachedPlateResolver) CacheStats() PlateCacheStats {
r.mu.Lock()
defer r.mu.Unlock()
return PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions}
}
func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (string, error) {
@@ -76,13 +106,46 @@ func (r *CachedPlateResolver) PlateByVIN(ctx context.Context, vin string) (strin
notFound: errors.Is(err, sql.ErrNoRows),
expiresAt: now.Add(r.ttl),
}
r.enforceLimitLocked(now)
stats := PlateCacheStats{Entries: len(r.entries), MaxEntries: r.maxEntries, Evictions: r.evictions}
observe := r.observe
r.mu.Unlock()
if observe != nil {
observe(stats)
}
if err != nil {
return "", err
}
return strings.TrimSpace(plate), nil
}
func (r *CachedPlateResolver) enforceLimitLocked(now time.Time) {
if r.maxEntries <= 0 || len(r.entries) <= r.maxEntries {
return
}
for vin, entry := range r.entries {
if now.After(entry.expiresAt) || now.Equal(entry.expiresAt) {
delete(r.entries, vin)
r.evictions++
}
}
for len(r.entries) > r.maxEntries {
victim := ""
var victimExpiresAt time.Time
for vin, entry := range r.entries {
if victim == "" || entry.expiresAt.Before(victimExpiresAt) {
victim = vin
victimExpiresAt = entry.expiresAt
}
}
if victim == "" {
return
}
delete(r.entries, victim)
r.evictions++
}
}
type SnapshotWriter struct {
exec SnapshotExecer
plateResolver PlateResolver
@@ -111,6 +174,19 @@ func (w *SnapshotWriter) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, realtimeLocationTableSQL); err != nil {
return err
}
for _, statement := range realtimeLocationCompatibilitySQL {
if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateColumnError(err) {
return err
}
}
for _, statement := range cleanupInvalidRealtimeTotalMileageSQL {
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
}
if _, err := w.exec.ExecContext(ctx, backfillRealtimeAccessProjectionSQL); err != nil {
return err
}
return nil
}
@@ -129,13 +205,11 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
if err != nil {
return err
}
eventTime := nullableTime(env.EventTimeMS)
eventTimeMS, _ := envelope.NormalizedEventTimeMS(env)
eventTime := nullableTime(eventTimeMS)
receivedAt := nullableTime(env.ReceivedAtMS)
platformName := platformNameFromEnvelope(env)
parsed, err := w.snapshotFieldsForEnvelope(ctx, env, vin)
if err != nil {
return err
}
parsed := snapshotFieldsForEnvelope(env)
if _, err = w.exec.ExecContext(ctx, upsertRealtimeSnapshotSQL,
string(env.Protocol),
vin,
@@ -146,6 +220,9 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
eventTime,
receivedAt,
env.StableEventID(),
receivedAt,
receivedAt,
env.StableEventID(),
); err != nil {
return err
}
@@ -162,6 +239,7 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
location.Longitude,
location.SpeedKMH,
location.TotalMileageKM,
location.TotalMileageAt,
location.SOCPercent,
location.AltitudeM,
location.DirectionDeg,
@@ -173,87 +251,25 @@ func (w *SnapshotWriter) Update(ctx context.Context, env envelope.FrameEnvelope)
return err
}
func (w *SnapshotWriter) snapshotFieldsForEnvelope(ctx context.Context, env envelope.FrameEnvelope, vin string) (map[string]any, error) {
if len(env.Parsed) == 0 {
return nil, nil
}
parsed := cloneMap(env.Parsed)
incoming := realtimeSnapshotFlatFields(env, parsed)
if queryer, ok := w.exec.(Queryer); ok {
existing, err := realtimeSnapshotParsedJSON(ctx, queryer, env.Protocol, vin)
if err != nil {
return nil, err
}
if len(existing) > 0 {
if isStructuredSnapshotParsed(env.Protocol, existing) {
parsed = mergeParsedForProtocol(env.Protocol, existing, env.Parsed)
mergedEnv := env
mergedEnv.ParsedFields = nil
mergedEnv.ParsedFieldTypes = nil
return realtimeSnapshotFlatFields(mergedEnv, parsed), nil
}
return mergeRealtimeSnapshotFields(existing, incoming), nil
}
}
return incoming, nil
}
func realtimeSnapshotFlatFields(env envelope.FrameEnvelope, parsed map[string]any) map[string]any {
if len(env.ParsedFields) > 0 {
fields := cloneMap(env.ParsedFields)
if len(fields) > 0 {
return fields
}
}
rows := realtimeKVFields(env, parsed)
if len(rows) == 0 {
func snapshotFieldsForEnvelope(env envelope.FrameEnvelope) map[string]any {
if len(env.ParsedFields) == 0 {
return nil
}
fields := make(map[string]any, len(rows))
for _, row := range rows {
key := realtimeKVFieldPath(row.Domain, row.Field)
if key == "" {
continue
}
fields[key] = row.Value
return realtimeSnapshotFlatFields(env)
}
func realtimeSnapshotFlatFields(env envelope.FrameEnvelope) map[string]any {
fields := cloneMap(env.ParsedFields)
filterInvalidRealtimeMeasurementFields(fields)
if len(fields) == 0 {
return nil
}
return fields
}
func mergeRealtimeSnapshotFields(existing map[string]any, incoming map[string]any) map[string]any {
if len(existing) == 0 {
return cloneMap(incoming)
}
merged := cloneMap(existing)
for key, value := range incoming {
merged[key] = value
}
return merged
}
func isStructuredSnapshotParsed(protocol envelope.Protocol, parsed map[string]any) bool {
if len(parsed) == 0 {
return false
}
if protocol == envelope.ProtocolGB32960 {
if _, ok := parsed["data_units"]; ok {
return true
}
}
mapping := realtimeMapping(protocol)
for key := range mapping.TopLevelName {
if _, ok := parsed[key]; ok {
return true
}
}
return false
}
func platformNameFromEnvelope(env envelope.FrameEnvelope) string {
if env.Fields != nil {
if value := strings.TrimSpace(stringValue(env.Fields["platform_account"])); value != "" {
return value
}
if value := strings.TrimSpace(env.PlatformName); value != "" {
return value
}
if env.Parsed != nil {
if value := strings.TrimSpace(stringValue(env.Parsed["platform_name"])); value != "" {
@@ -275,25 +291,6 @@ func stringValue(value any) string {
}
}
func realtimeSnapshotParsedJSON(ctx context.Context, queryer Queryer, protocol envelope.Protocol, vin string) (map[string]any, error) {
var raw sql.NullString
err := queryer.QueryRowContext(ctx, selectRealtimeSnapshotParsedJSONSQL, string(protocol), vin).Scan(&raw)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, err
}
if !raw.Valid || strings.TrimSpace(raw.String) == "" {
return nil, nil
}
var parsed map[string]any
if err := json.Unmarshal([]byte(raw.String), &parsed); err != nil {
return nil, nil
}
return parsed, nil
}
func marshalParsedJSON(parsed map[string]any) any {
if len(parsed) == 0 {
return nil
@@ -335,6 +332,7 @@ type realtimeLocationRow struct {
Longitude float64
SpeedKMH any
TotalMileageKM any
TotalMileageAt any
SOCPercent any
AltitudeM any
DirectionDeg any
@@ -345,71 +343,53 @@ type realtimeLocationRow struct {
}
func realtimeLocationFromEnvelope(env envelope.FrameEnvelope, vin string, plate string) (realtimeLocationRow, bool) {
latitude, okLat := numberField(env.Fields, envelope.FieldLatitude)
longitude, okLon := numberField(env.Fields, envelope.FieldLongitude)
if !okLat || !okLon {
location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
if !ok {
return realtimeLocationRow{}, false
}
eventTimeMS, _ := envelope.NormalizedEventTimeMS(env)
totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
if totalMileageKM <= 0 {
hasTotalMileage = false
}
var totalMileageValue any
var totalMileageAt any
if hasTotalMileage {
totalMileageValue = totalMileageKM
totalMileageAt = nullableTime(eventTimeMS)
}
return realtimeLocationRow{
Protocol: string(env.Protocol),
VIN: vin,
Plate: plate,
EventTime: nullableTime(env.EventTimeMS),
Latitude: latitude,
Longitude: longitude,
SpeedKMH: nullableNumberField(env.Fields, envelope.FieldSpeedKMH),
TotalMileageKM: nullableNumberField(env.Fields, envelope.FieldTotalMileageKM),
SOCPercent: nullableNumberField(env.Fields, envelope.FieldSOCPercent),
AltitudeM: nullableNumberField(env.Fields, "altitude_m"),
DirectionDeg: nullableNumberField(env.Fields, "direction_deg"),
AlarmFlag: nullableNumberField(env.Fields, "alarm_flag"),
StatusFlag: nullableNumberField(env.Fields, "status_flag"),
EventTime: nullableTime(eventTimeMS),
Latitude: location.Latitude,
Longitude: location.Longitude,
SpeedKMH: optionalFloatValue(location.SpeedKMH),
TotalMileageKM: totalMileageValue,
TotalMileageAt: totalMileageAt,
SOCPercent: optionalFloatValue(location.SOCPercent),
AltitudeM: optionalFloatValue(location.AltitudeM),
DirectionDeg: optionalFloatValue(location.DirectionDeg),
AlarmFlag: optionalInt64Value(location.AlarmFlag),
StatusFlag: optionalInt64Value(location.StatusFlag),
ReceivedAt: nullableTime(env.ReceivedAtMS),
EventID: env.StableEventID(),
}, true
}
func nullableNumberField(fields map[string]any, key string) any {
value, ok := numberField(fields, key)
if !ok {
func optionalFloatValue(value *float64) any {
if value == nil {
return nil
}
return value
return *value
}
func numberField(fields map[string]any, key string) (float64, bool) {
value, ok := fields[key]
if !ok {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int8:
return float64(typed), true
case int16:
return float64(typed), true
case int32:
return float64(typed), true
case int64:
return float64(typed), true
case uint:
return float64(typed), true
case uint8:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case uint64:
return float64(typed), true
default:
return 0, false
func optionalInt64Value(value *int64) any {
if value == nil {
return nil
}
return *value
}
type BindingPlateResolver struct {
@@ -469,30 +449,19 @@ func nullableTime(ms int64) any {
func isRealtimeSnapshotEvent(env envelope.FrameEnvelope) bool {
switch env.Protocol {
case envelope.ProtocolGB32960:
return env.MessageID == "0x02" && hasGB32960RealtimeDataUnits(env)
return (env.MessageID == "0x02" || env.MessageID == "0x03") && telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields)
case envelope.ProtocolJT808:
return env.MessageID == "0x0200" && hasLocationFields(env)
case envelope.ProtocolYutongMQTT:
return len(env.Fields) > 0
return telemetry.HasRealtimeFields(env.Protocol, env.ParsedFields)
default:
return false
}
}
func hasGB32960RealtimeDataUnits(env envelope.FrameEnvelope) bool {
if units, ok := env.Parsed["data_units"].([]any); ok && len(units) > 0 {
return true
}
if units, ok := env.Parsed["data_units"].([]map[string]any); ok && len(units) > 0 {
return true
}
return false
}
func hasLocationFields(env envelope.FrameEnvelope) bool {
_, okLat := numberField(env.Fields, envelope.FieldLatitude)
_, okLon := numberField(env.Fields, envelope.FieldLongitude)
return okLat && okLon
_, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
return ok
}
const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot (
@@ -505,16 +474,31 @@ const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_sn
event_time DATETIME(3) NULL,
received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '',
access_first_seen_at DATETIME(3) NULL,
access_previous_received_at DATETIME(3) NULL,
access_latest_received_at DATETIME(3) NULL,
access_report_interval_ms BIGINT UNSIGNED NULL,
access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0,
access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '',
access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '',
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (protocol, vin),
KEY idx_vin (vin),
KEY idx_protocol_updated (protocol, updated_at)
KEY idx_protocol_updated (protocol, updated_at),
KEY idx_access_latest (access_latest_received_at, vin)
)`
var realtimeSnapshotCompatibilitySQL = []string{
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN platform_name VARCHAR(64) NOT NULL DEFAULT '' AFTER plate",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN peer VARCHAR(128) NOT NULL DEFAULT '' AFTER platform_name",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json LONGTEXT NULL AFTER peer",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_at DATETIME(3) NULL AFTER event_id",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_previous_received_at DATETIME(3) NULL AFTER access_first_seen_at",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_received_at DATETIME(3) NULL AFTER access_previous_received_at",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_report_interval_ms BIGINT UNSIGNED NULL AFTER access_latest_received_at",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_sample_count BIGINT UNSIGNED NOT NULL DEFAULT 0 AFTER access_report_interval_ms",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_latest_event_id VARCHAR(64) NOT NULL DEFAULT '' AFTER access_sample_count",
"ALTER TABLE vehicle_realtime_snapshot ADD COLUMN access_first_seen_source VARCHAR(32) NOT NULL DEFAULT '' AFTER access_latest_event_id",
}
func isDuplicateColumnError(err error) bool {
@@ -524,21 +508,27 @@ func isDuplicateColumnError(err error) bool {
const upsertRealtimeSnapshotSQL = `
INSERT INTO vehicle_realtime_snapshot
(protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
(protocol, vin, plate, platform_name, peer, parsed_json, event_time, received_at, event_id,
access_first_seen_at, access_latest_received_at, access_sample_count, access_latest_event_id, access_first_seen_source)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, 'live_writer')
ON DUPLICATE KEY UPDATE
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
platform_name = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(platform_name), platform_name),
peer = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(peer), peer),
parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(parsed_json), parsed_json),
parsed_json = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time),
IF(VALUES(parsed_json) IS NULL OR VALUES(parsed_json) = '', parsed_json, JSON_MERGE_PATCH(COALESCE(NULLIF(vehicle_realtime_snapshot.parsed_json, ''), JSON_OBJECT()), VALUES(parsed_json))),
parsed_json),
event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_time), event_time),
received_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(received_at), received_at),
event_id = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), VALUES(event_id), event_id),
updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at)
updated_at = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_snapshot.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_snapshot.event_time), CURRENT_TIMESTAMP, updated_at),
access_previous_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, vehicle_realtime_snapshot.access_latest_received_at, access_previous_received_at),
access_report_interval_ms = IF(VALUES(access_latest_received_at) IS NOT NULL AND vehicle_realtime_snapshot.access_latest_received_at IS NOT NULL AND VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, TIMESTAMPDIFF(MICROSECOND, vehicle_realtime_snapshot.access_latest_received_at, VALUES(access_latest_received_at)) DIV 1000, access_report_interval_ms),
access_sample_count = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, access_sample_count + 1, access_sample_count),
access_latest_received_at = IF(VALUES(access_latest_received_at) IS NOT NULL AND (vehicle_realtime_snapshot.access_latest_received_at IS NULL OR VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at) AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_received_at), access_latest_received_at),
access_latest_event_id = IF(VALUES(access_latest_received_at) IS NOT NULL AND VALUES(access_latest_received_at) = vehicle_realtime_snapshot.access_latest_received_at AND VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id, VALUES(access_latest_event_id), access_latest_event_id)
`
const selectRealtimeSnapshotParsedJSONSQL = `SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = ? AND vin = ?`
const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location (
protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
@@ -548,6 +538,7 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
longitude DECIMAL(12,6) NOT NULL,
speed_kmh DECIMAL(10,3) NULL,
total_mileage_km DECIMAL(18,3) NULL,
total_mileage_event_time DATETIME(3) NULL,
soc_percent DECIMAL(6,2) NULL,
altitude_m DECIMAL(10,3) NULL,
direction_deg DECIMAL(10,3) NULL,
@@ -561,11 +552,50 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
KEY idx_protocol_updated (protocol, updated_at)
)`
var realtimeLocationCompatibilitySQL = []string{
"ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time DATETIME(3) NULL AFTER total_mileage_km",
}
var cleanupInvalidRealtimeTotalMileageSQL = []string{
`UPDATE vehicle_realtime_location
SET total_mileage_km = NULL,
total_mileage_event_time = NULL
WHERE total_mileage_km IS NOT NULL
AND total_mileage_km <= 0`,
`UPDATE vehicle_realtime_snapshot
SET parsed_json = JSON_REMOVE(parsed_json, '$."jt808.location.total_mileage_km"')
WHERE protocol = 'JT808'
AND parsed_json IS NOT NULL
AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."jt808.location.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
`UPDATE vehicle_realtime_snapshot
SET parsed_json = JSON_REMOVE(parsed_json, '$."gb32960.vehicle.total_mileage_km"')
WHERE protocol = 'GB32960'
AND parsed_json IS NOT NULL
AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."gb32960.vehicle.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
`UPDATE vehicle_realtime_snapshot
SET parsed_json = JSON_REMOVE(parsed_json, '$."yutong_mqtt.data.total_mileage_km"')
WHERE protocol = 'YUTONG_MQTT'
AND parsed_json IS NOT NULL
AND CAST(JSON_UNQUOTE(JSON_EXTRACT(parsed_json, '$."yutong_mqtt.data.total_mileage_km"')) AS DECIMAL(18,3)) <= 0`,
}
const backfillRealtimeAccessProjectionSQL = `UPDATE vehicle_realtime_snapshot
SET access_first_seen_at = COALESCE(access_first_seen_at, received_at, updated_at),
access_latest_received_at = COALESCE(access_latest_received_at, received_at, updated_at),
access_sample_count = IF(access_sample_count = 0, 1, access_sample_count),
access_latest_event_id = IF(access_latest_event_id = '', event_id, access_latest_event_id),
access_first_seen_source = IF(access_first_seen_source = '', 'snapshot_backfill', access_first_seen_source)
WHERE access_first_seen_at IS NULL
OR access_latest_received_at IS NULL
OR access_sample_count = 0
OR access_latest_event_id = ''
OR access_first_seen_source = ''`
const upsertRealtimeLocationSQL = `
INSERT INTO vehicle_realtime_location
(protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km,
(protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, total_mileage_event_time,
soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
plate = IF(VALUES(plate) <> '', VALUES(plate), plate),
event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(event_time), event_time),
@@ -573,6 +603,7 @@ ON DUPLICATE KEY UPDATE
longitude = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), VALUES(longitude), longitude),
speed_kmh = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(speed_kmh), speed_kmh), speed_kmh),
total_mileage_km = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(total_mileage_km), total_mileage_km), total_mileage_km),
total_mileage_event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), IF(VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), total_mileage_event_time),
soc_percent = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(soc_percent), soc_percent), soc_percent),
altitude_m = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(altitude_m), altitude_m), altitude_m),
direction_deg = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), COALESCE(VALUES(direction_deg), direction_deg), direction_deg),

459
go/vehicle-gateway/internal/realtime/snapshot_writer_test.go
View File

@@ -37,26 +37,30 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) {
map[string]any{"name": "gd_fc_vendor_tlv", "type": "vendor", "value": map[string]any{"foo": "bar"}},
},
},
ParsedFields: map[string]any{
"gb32960.vehicle.soc_percent": "90",
"gb32960.gd_fc_vendor_tlv.foo": "bar",
},
Fields: map[string]any{envelope.FieldSOCPercent: 90},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if len(exec.calls) != 6 {
t.Fatalf("exec calls = %d, want 6", len(exec.calls))
if len(exec.calls) != 19 {
t.Fatalf("exec calls = %d, want 19", len(exec.calls))
}
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot") {
t.Fatalf("schema query = %s", exec.calls[0].query)
}
if !strings.Contains(exec.calls[4].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_location") {
t.Fatalf("location schema query = %s", exec.calls[4].query)
if !strings.Contains(exec.calls[11].query, "CREATE TABLE IF NOT EXISTS vehicle_realtime_location") {
t.Fatalf("location schema query = %s", exec.calls[11].query)
}
for _, call := range exec.calls {
if strings.Contains(call.query, "vehicle_realtime_kv") {
t.Fatalf("snapshot writer should not create or write MySQL realtime kv: %s", call.query)
}
}
for _, call := range []snapshotExecCall{exec.calls[0], exec.calls[4]} {
for _, call := range []snapshotExecCall{exec.calls[0], exec.calls[11]} {
if strings.Contains(call.query, "fields_json") {
t.Fatalf("realtime schema should not contain fields_json: %s", call.query)
}
@@ -74,10 +78,21 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) {
t.Fatalf("snapshot schema should contain %s: %s", want, exec.calls[0].query)
}
}
if strings.Contains(exec.calls[4].query, "idx_location") {
t.Fatalf("realtime location table should not keep unused geo index: %s", exec.calls[4].query)
if strings.Contains(exec.calls[11].query, "idx_location") {
t.Fatalf("realtime location table should not keep unused geo index: %s", exec.calls[11].query)
}
upsert := exec.calls[5]
if !strings.Contains(exec.calls[12].query, "total_mileage_event_time") {
t.Fatalf("location compatibility migration should add total mileage event time: %s", exec.calls[12].query)
}
for _, call := range exec.calls[13:17] {
if !strings.Contains(call.query, "total_mileage") {
t.Fatalf("schema bootstrap should clean invalid realtime mileage: %s", call.query)
}
}
if !strings.Contains(exec.calls[17].query, "snapshot_backfill") {
t.Fatalf("access projection baseline should be backfilled explicitly: %s", exec.calls[17].query)
}
upsert := exec.calls[18]
if !strings.Contains(upsert.query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("upsert query = %s", upsert.query)
}
@@ -101,8 +116,13 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) {
if got := upsert.args[5]; !strings.Contains(got.(string), `"gb32960.vehicle.soc_percent":"90"`) {
t.Fatalf("parsed json arg = %#v", got)
}
if len(upsert.args) != 9 {
t.Fatalf("snapshot upsert args = %d, want 9", len(upsert.args))
if len(upsert.args) != 12 {
t.Fatalf("snapshot upsert args = %d, want 12", len(upsert.args))
}
for _, want := range []string{"access_first_seen_at", "access_previous_received_at", "access_latest_received_at", "access_report_interval_ms", "access_sample_count", "access_latest_event_id", "live_writer", "TIMESTAMPDIFF(MICROSECOND"} {
if !strings.Contains(upsert.query, want) {
t.Fatalf("access projection upsert missing %q: %s", want, upsert.query)
}
}
}
@@ -122,8 +142,24 @@ func TestSnapshotWriterEnsureSchemaOnlyCreatesTargetTables(t *testing.T) {
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("ALTER TABLE vehicle_realtime_snapshot ADD COLUMN parsed_json").
WillReturnResult(sqlmock.NewResult(0, 0))
for _, column := range []string{"access_first_seen_at", "access_previous_received_at", "access_latest_received_at", "access_report_interval_ms", "access_sample_count", "access_latest_event_id", "access_first_seen_source"} {
mock.ExpectExec("ALTER TABLE vehicle_realtime_snapshot ADD COLUMN " + column).
WillReturnResult(sqlmock.NewResult(0, 0))
}
mock.ExpectExec("CREATE TABLE IF NOT EXISTS vehicle_realtime_location").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("ALTER TABLE vehicle_realtime_location ADD COLUMN total_mileage_event_time").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_realtime_location").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_realtime_snapshot").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_realtime_snapshot").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_realtime_snapshot").
WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("UPDATE vehicle_realtime_snapshot").
WillReturnResult(sqlmock.NewResult(0, 0))
if err := writer.EnsureSchema(context.Background()); err != nil {
t.Fatalf("EnsureSchema() error = %v", err)
@@ -133,6 +169,41 @@ func TestSnapshotWriterEnsureSchemaOnlyCreatesTargetTables(t *testing.T) {
}
}
func TestRealtimeAccessProjectionSQLProtectsDuplicateAndOutOfOrderReceipts(t *testing.T) {
accessStart := strings.Index(upsertRealtimeSnapshotSQL, "access_previous_received_at =")
if accessStart < 0 {
t.Fatal("access projection assignments are missing")
}
accessSQL := upsertRealtimeSnapshotSQL[accessStart:]
for _, want := range []string{
"VALUES(access_latest_received_at) > vehicle_realtime_snapshot.access_latest_received_at",
"VALUES(access_latest_event_id) <> vehicle_realtime_snapshot.access_latest_event_id",
"TIMESTAMPDIFF(MICROSECOND",
"access_sample_count + 1",
} {
if !strings.Contains(accessSQL, want) {
t.Fatalf("access projection SQL missing %q:\n%s", want, accessSQL)
}
}
if strings.Contains(accessSQL, "VALUES(event_time)") {
t.Fatalf("access receipt projection must not be gated by device event time:\n%s", accessSQL)
}
ordered := []string{"access_previous_received_at =", "access_report_interval_ms =", "access_sample_count =", "access_latest_received_at =", "access_latest_event_id ="}
previous := -1
for _, assignment := range ordered {
position := strings.Index(accessSQL, assignment)
if position <= previous {
t.Fatalf("access assignment %q must preserve old latest receipt before advancing it: %s", assignment, accessSQL)
}
previous = position
}
for _, want := range []string{"COALESCE(access_first_seen_at, received_at, updated_at)", "snapshot_backfill", "access_sample_count = 0"} {
if !strings.Contains(backfillRealtimeAccessProjectionSQL, want) {
t.Fatalf("access baseline backfill missing %q: %s", want, backfillRealtimeAccessProjectionSQL)
}
}
}
func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T) {
exec := &recordingSnapshotExec{}
writer := NewSnapshotWriter(exec)
@@ -150,16 +221,15 @@ func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T)
ReceivedAtMS: 1782918601000,
EventID: "event-1",
Parsed: map[string]any{"location": map[string]any{"latitude": 30.590151, "longitude": 121.069881}},
Fields: map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10241.2,
envelope.FieldSOCPercent: 88,
"altitude_m": uint16(5),
"direction_deg": uint16(79),
"alarm_flag": uint32(0),
"status_flag": uint32(786435),
ParsedFields: map[string]any{
"jt808.location.latitude": 30.590151,
"jt808.location.longitude": 121.069881,
"jt808.location.speed_kmh": 23.0,
"jt808.location.total_mileage_km": "10241.2",
"jt808.location.altitude_m": uint16(5),
"jt808.location.direction_deg": uint16(79),
"jt808.location.alarm_flag": uint32(0),
"jt808.location.status_flag": uint32(786435),
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
@@ -198,15 +268,190 @@ func TestSnapshotWriterUpsertsRealtimeLocationWhenCoordinatesExist(t *testing.T)
if got, want := locationUpsert.args[7], 10241.2; got != want {
t.Fatalf("mileage arg = %#v, want %v", got, want)
}
if got, want := locationUpsert.args[8], 88.0; got != want {
t.Fatalf("soc arg = %#v, want %v", got, want)
if _, ok := locationUpsert.args[8].(time.Time); !ok {
t.Fatalf("total mileage event time arg = %#v, want time.Time", locationUpsert.args[8])
}
if len(locationUpsert.args) != 15 {
t.Fatalf("location upsert args = %d, want 15", len(locationUpsert.args))
if got := locationUpsert.args[9]; got != nil {
t.Fatalf("JT808 location should not invent SOC, got %#v", got)
}
if len(locationUpsert.args) != 16 {
t.Fatalf("location upsert args = %d, want 16", len(locationUpsert.args))
}
}
func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testing.T) {
func TestSnapshotWriterDropsNonPositiveTotalMileageFromRealtimeStores(t *testing.T) {
exec := &recordingSnapshotExec{}
writer := NewSnapshotWriter(exec)
if err := writer.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
ParsedFields: map[string]any{
"jt808.location.latitude": "30.590151",
"jt808.location.longitude": "121.069881",
"jt808.location.speed_kmh": "23",
"jt808.location.total_mileage_km": "0",
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if len(exec.calls) != 2 {
t.Fatalf("exec calls = %d, want snapshot + location", len(exec.calls))
}
parsedJSON, ok := exec.calls[0].args[5].(string)
if !ok {
t.Fatalf("snapshot parsed_json arg = %#v", exec.calls[0].args[5])
}
if strings.Contains(parsedJSON, "total_mileage_km") {
t.Fatalf("snapshot parsed_json should drop non-positive mileage: %s", parsedJSON)
}
if !strings.Contains(parsedJSON, "jt808.location.speed_kmh") {
t.Fatalf("snapshot parsed_json should keep valid fields: %s", parsedJSON)
}
if exec.calls[1].args[7] != nil {
t.Fatalf("location total mileage arg = %#v, want nil", exec.calls[1].args[7])
}
if exec.calls[1].args[8] != nil {
t.Fatalf("location total mileage time arg = %#v, want nil", exec.calls[1].args[8])
}
}
func TestSnapshotWriterNormalizesFarFutureEventTime(t *testing.T) {
exec := &recordingSnapshotExec{}
writer := NewSnapshotWriter(exec)
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC)
futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC)
if err := writer.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
MessageID: "0x0200",
VIN: "VIN001",
EventTimeMS: futureEvent.UnixMilli(),
ReceivedAtMS: received.UnixMilli(),
Parsed: map[string]any{"location": map[string]any{"latitude": 30.590151, "longitude": 121.069881}},
ParsedFields: map[string]any{
"jt808.location.latitude": 30.590151,
"jt808.location.longitude": 121.069881,
"jt808.location.total_mileage_km": "10241.2",
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if len(exec.calls) != 2 {
t.Fatalf("exec calls = %d, want snapshot + location", len(exec.calls))
}
if got, ok := exec.calls[0].args[6].(time.Time); !ok || !got.Equal(received) {
t.Fatalf("snapshot event_time arg = %#v, want received %s", exec.calls[0].args[6], received)
}
if got, ok := exec.calls[1].args[3].(time.Time); !ok || !got.Equal(received) {
t.Fatalf("location event_time arg = %#v, want received %s", exec.calls[1].args[3], received)
}
if got, ok := exec.calls[1].args[8].(time.Time); !ok || !got.Equal(received) {
t.Fatalf("total mileage event time arg = %#v, want received %s", exec.calls[1].args[8], received)
}
}
func TestRealtimeLocationUsesProtocolMileageMapping(t *testing.T) {
tests := []struct {
name string
protocol envelope.Protocol
fields map[string]any
wantMileage float64
}{
{
name: "gb32960 kilometers",
protocol: envelope.ProtocolGB32960,
fields: map[string]any{
"gb32960.position.latitude": 30.590151,
"gb32960.position.longitude": 121.069881,
"gb32960.vehicle.total_mileage_km": "4123.9",
},
wantMileage: 4123.9,
},
{
name: "jt808 kilometers",
protocol: envelope.ProtocolJT808,
fields: map[string]any{
"jt808.location.latitude": 30.590151,
"jt808.location.longitude": 121.069881,
"jt808.location.total_mileage_km": json.Number("10241.2"),
},
wantMileage: 10241.2,
},
{
name: "yutong meters",
protocol: envelope.ProtocolYutongMQTT,
fields: map[string]any{
"yutong_mqtt.data.latitude": 30.590151,
"yutong_mqtt.data.longitude": 121.069881,
"yutong_mqtt.data.total_mileage": "86737000",
},
wantMileage: 86737,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
row, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{
Protocol: tc.protocol,
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
ParsedFields: tc.fields,
}, "VIN001", "")
if !ok {
t.Fatal("realtimeLocationFromEnvelope() should produce a location")
}
if got, ok := row.TotalMileageKM.(float64); !ok || got != tc.wantMileage {
t.Fatalf("total mileage = %#v, want %v", row.TotalMileageKM, tc.wantMileage)
}
if _, ok := row.TotalMileageAt.(time.Time); !ok {
t.Fatalf("total mileage event time = %#v, want time.Time", row.TotalMileageAt)
}
})
}
}
func TestRealtimeLocationDoesNotAdvanceMileageFromCoreFieldOnSparseFrame(t *testing.T) {
row, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolYutongMQTT,
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
ParsedFields: map[string]any{
"yutong_mqtt.data.latitude": "30.590151",
"yutong_mqtt.data.longitude": "121.069881",
},
}, "VIN001", "")
if !ok {
t.Fatal("realtimeLocationFromEnvelope() should produce a location")
}
if row.TotalMileageKM != nil {
t.Fatalf("sparse frame total mileage = %#v, want nil", row.TotalMileageKM)
}
if row.TotalMileageAt != nil {
t.Fatalf("sparse frame total mileage event time = %#v, want nil", row.TotalMileageAt)
}
}
func TestRealtimeLocationRejectsBareStandardizedFields(t *testing.T) {
_, ok := realtimeLocationFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Fields: map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
},
}, "VIN001", "")
if ok {
t.Fatal("bare standardized fields must not drive the canonical location projection")
}
}
func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFramesInUpsert(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatal(err)
@@ -214,10 +459,6 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi
defer db.Close()
writer := NewSnapshotWriter(db)
existing := `{"data_units":[{"type":"0x01","name":"vehicle","value":{"soc_percent":88}}]}`
mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?").
WithArgs("GB32960", "VIN001").
WillReturnRows(sqlmock.NewRows([]string{"parsed_json"}).AddRow(existing))
mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot").
WithArgs(
"GB32960",
@@ -226,14 +467,15 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi
"",
"",
jsonFlatFieldsArg{fields: map[string]string{
"gb32960.vehicle.soc_percent": "88",
"gb32960.vehicle.type": "0x01",
"gb32960.gd_fc_stack.stack_count": "1",
"gb32960.gd_fc_stack.type": "0x30",
}},
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
).
WillReturnResult(sqlmock.NewResult(0, 1))
@@ -248,6 +490,10 @@ func TestSnapshotWriterMergesGB32960ParsedJSONAcrossSplitRealtimeFrames(t *testi
map[string]any{"type": "0x30", "name": "gd_fc_stack", "value": map[string]any{"stack_count": 1}},
},
},
ParsedFields: map[string]any{
"gb32960.gd_fc_stack.stack_count": "1",
"gb32960.gd_fc_stack.type": "0x30",
},
})
if err != nil {
t.Fatalf("Update() error = %v", err)
@@ -265,9 +511,6 @@ func TestSnapshotWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T)
defer db.Close()
writer := NewSnapshotWriter(db)
mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?").
WithArgs("GB32960", "VIN001").
WillReturnRows(sqlmock.NewRows([]string{"parsed_json"}))
mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot").
WithArgs(
"GB32960",
@@ -281,6 +524,9 @@ func TestSnapshotWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T)
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
).
WillReturnResult(sqlmock.NewResult(0, 1))
@@ -316,10 +562,6 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te
defer db.Close()
writer := NewSnapshotWriter(db)
existing := `{"data_units":[{"type":"0x30","name":"gd_fc_stack","value":{"stack_count":1,"summaries":[{"cell_count":432,"stack_water_outlet_temp_c":65,"frame_cell_start":201,"frame_cell_count":200}]}}]}`
mock.ExpectQuery("SELECT parsed_json FROM vehicle_realtime_snapshot WHERE protocol = \\? AND vin = \\?").
WithArgs("GB32960", "VIN001").
WillReturnRows(sqlmock.NewRows([]string{"parsed_json"}).AddRow(existing))
mock.ExpectExec("INSERT INTO vehicle_realtime_snapshot").
WithArgs(
"GB32960",
@@ -341,6 +583,9 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
sqlmock.AnyArg(),
).
WillReturnResult(sqlmock.NewResult(0, 1))
@@ -364,6 +609,9 @@ func TestSnapshotWriterCollapsesGB32960StackFragmentsWhenMergingParsedJSON(t *te
},
},
},
ParsedFields: map[string]any{
"gb32960.gd_fc_stack.stack_water_outlet_temp_c": "63",
},
})
if err != nil {
t.Fatalf("Update() error = %v", err)
@@ -397,6 +645,18 @@ func TestRealtimeUpsertsDoNotOverwriteWithOlderEventTime(t *testing.T) {
}
}
func TestRealtimeSnapshotUpsertMergesParsedJSONInSQL(t *testing.T) {
for _, want := range []string{
"JSON_MERGE_PATCH",
"COALESCE(NULLIF(vehicle_realtime_snapshot.parsed_json, ''), JSON_OBJECT())",
"VALUES(parsed_json)",
} {
if !strings.Contains(upsertRealtimeSnapshotSQL, want) {
t.Fatalf("snapshot upsert should merge parsed_json in SQL, missing %q:\n%s", want, upsertRealtimeSnapshotSQL)
}
}
}
func TestRealtimeLocationUpsertKeepsExistingSparseFieldsWhenMQTTOnlySendsCoordinates(t *testing.T) {
for _, column := range []string{
"speed_kmh",
@@ -412,6 +672,10 @@ func TestRealtimeLocationUpsertKeepsExistingSparseFieldsWhenMQTTOnlySendsCoordin
t.Fatalf("location upsert should keep existing %s when incoming sparse MQTT frame omits it; missing:\n%s\nin:\n%s", column, want, upsertRealtimeLocationSQL)
}
}
wantMileageAt := "total_mileage_event_time = IF(VALUES(event_time) IS NOT NULL AND (vehicle_realtime_location.event_time IS NULL OR VALUES(event_time) >= vehicle_realtime_location.event_time), IF(VALUES(total_mileage_km) IS NOT NULL, COALESCE(VALUES(total_mileage_event_time), total_mileage_event_time), total_mileage_event_time), total_mileage_event_time)"
if !strings.Contains(upsertRealtimeLocationSQL, wantMileageAt) {
t.Fatalf("location upsert should only advance total_mileage_event_time when the incoming frame carries mileage; missing:\n%s\nin:\n%s", wantMileageAt, upsertRealtimeLocationSQL)
}
}
func TestSnapshotWriterBackfillsPlateFromBindingByVIN(t *testing.T) {
@@ -426,10 +690,7 @@ func TestSnapshotWriterBackfillsPlateFromBindingByVIN(t *testing.T) {
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Parsed: sampleGB32960RealtimeParsed(),
Fields: map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
},
ParsedFields: sampleGB32960LocationFields(),
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
@@ -481,10 +742,7 @@ func TestSnapshotWriterCachesBindingPlateByVIN(t *testing.T) {
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Parsed: sampleGB32960RealtimeParsed(),
Fields: map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
},
ParsedFields: sampleGB32960LocationFields(),
}
if err := writer.Update(context.Background(), event); err != nil {
@@ -509,6 +767,61 @@ func TestSnapshotWriterCachesBindingPlateByVIN(t *testing.T) {
}
}
func TestCachedPlateResolverEvictsOldestEntryWhenLimitExceeded(t *testing.T) {
delegate := &mapPlateResolver{plates: map[string]string{
"VIN001": "沪A00001",
"VIN002": "沪A00002",
"VIN003": "沪A00003",
}}
resolver := NewCachedPlateResolverWithMaxEntries(delegate, time.Hour, 2)
now := time.Date(2026, 7, 12, 10, 0, 0, 0, time.UTC)
resolver.now = func() time.Time { return now }
var observed PlateCacheStats
resolver.SetStatsObserver(func(stats PlateCacheStats) {
observed = stats
})
for _, vin := range []string{"VIN001", "VIN002", "VIN003"} {
if _, err := resolver.PlateByVIN(context.Background(), vin); err != nil {
t.Fatalf("PlateByVIN(%s) error = %v", vin, err)
}
now = now.Add(time.Second)
}
stats := resolver.CacheStats()
if stats.Entries != 2 || stats.MaxEntries != 2 || stats.Evictions != 1 {
t.Fatalf("cache stats = %+v, want entries=2 max=2 evictions=1", stats)
}
if observed != stats {
t.Fatalf("observed stats = %+v, want %+v", observed, stats)
}
if _, ok := resolver.entries["VIN001"]; ok {
t.Fatalf("oldest VIN should be evicted")
}
if _, ok := resolver.entries["VIN002"]; !ok {
t.Fatalf("VIN002 should remain cached")
}
if _, ok := resolver.entries["VIN003"]; !ok {
t.Fatalf("VIN003 should remain cached")
}
}
func TestCachedPlateResolverCanDisableEntryLimit(t *testing.T) {
delegate := &mapPlateResolver{plates: map[string]string{
"VIN001": "沪A00001",
"VIN002": "沪A00002",
}}
resolver := NewCachedPlateResolverWithMaxEntries(delegate, time.Hour, 0)
for _, vin := range []string{"VIN001", "VIN002"} {
if _, err := resolver.PlateByVIN(context.Background(), vin); err != nil {
t.Fatalf("PlateByVIN(%s) error = %v", vin, err)
}
}
stats := resolver.CacheStats()
if stats.Entries != 2 || stats.MaxEntries != 0 || stats.Evictions != 0 {
t.Fatalf("cache stats = %+v, want unlimited cache with two entries", stats)
}
}
func TestSnapshotWriterKeepsEventPlateWhenPresent(t *testing.T) {
exec := &recordingSnapshotExec{}
resolver := &recordingPlateResolver{plate: "沪B99999"}
@@ -521,9 +834,9 @@ func TestSnapshotWriterKeepsEventPlateWhenPresent(t *testing.T) {
Plate: "沪A12345",
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Fields: map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
ParsedFields: map[string]any{
"jt808.location.latitude": 30.590151,
"jt808.location.longitude": 121.069881,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
@@ -548,7 +861,7 @@ func TestSnapshotWriterIgnoresMissingBindingPlate(t *testing.T) {
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Parsed: sampleGB32960RealtimeParsed(),
Fields: map[string]any{envelope.FieldSOCPercent: 90},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 90},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
@@ -568,7 +881,7 @@ func TestSnapshotWriterReturnsUnexpectedPlateLookupError(t *testing.T) {
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
Parsed: sampleGB32960RealtimeParsed(),
Fields: map[string]any{envelope.FieldSOCPercent: 90},
ParsedFields: map[string]any{"gb32960.vehicle.soc_percent": 90},
})
if err == nil || !strings.Contains(err.Error(), "db down") {
t.Fatalf("Update() error = %v, want db down", err)
@@ -701,6 +1014,9 @@ func TestSnapshotWriterSkipsMQTTWithoutActualDataFields(t *testing.T) {
"topic": "/vehicle/VIN001/state",
"data": map[string]any{},
},
ParsedFields: map[string]any{
"yutong_mqtt.metadata.topic": "/vehicle/VIN001/state",
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
@@ -709,6 +1025,27 @@ func TestSnapshotWriterSkipsMQTTWithoutActualDataFields(t *testing.T) {
}
}
func TestSnapshotWriterAcceptsGB32960RetransmissionFields(t *testing.T) {
exec := &recordingSnapshotExec{}
writer := NewSnapshotWriter(exec)
if err := writer.Update(context.Background(), envelope.FrameEnvelope{
Protocol: envelope.ProtocolGB32960,
MessageID: "0x03",
VIN: "VIN001",
EventTimeMS: 1782918600000,
ReceivedAtMS: 1782918601000,
ParsedFields: map[string]any{
"gb32960.vehicle.soc_percent": 90,
},
}); err != nil {
t.Fatalf("Update() error = %v", err)
}
if len(exec.calls) != 1 || !strings.Contains(exec.calls[0].query, "vehicle_realtime_snapshot") {
t.Fatalf("GB32960 retransmission should update snapshot, calls=%#v", exec.calls)
}
}
func TestSnapshotWriterSkipsEmptyDataUnitsWithoutCoreFields(t *testing.T) {
exec := &recordingSnapshotExec{}
writer := NewSnapshotWriter(exec)
@@ -740,6 +1077,14 @@ func sampleGB32960RealtimeParsed() map[string]any {
}
}
func sampleGB32960LocationFields() map[string]any {
return map[string]any{
"gb32960.position.latitude": 30.590151,
"gb32960.position.longitude": 121.069881,
"gb32960.vehicle.soc_percent": 90,
}
}
type snapshotExecCall struct {
query string
args []any
@@ -763,6 +1108,18 @@ func (r *recordingPlateResolver) PlateByVIN(_ context.Context, vin string) (stri
return r.plate, r.err
}
type mapPlateResolver struct {
plates map[string]string
}
func (r *mapPlateResolver) PlateByVIN(_ context.Context, vin string) (string, error) {
plate := strings.TrimSpace(r.plates[strings.TrimSpace(vin)])
if plate == "" {
return "", sql.ErrNoRows
}
return plate, nil
}
type jsonFlatFieldsArg struct {
fields map[string]string
forbidden []string