shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

235
go/vehicle-gateway/internal/history/writer_test.go
View File

@@ -3,6 +3,8 @@ package history
import (
"context"
"database/sql"
"errors"
"strconv"
"strings"
"sync"
"testing"
@@ -32,6 +34,18 @@ func TestSchemaStatementsCreateCoreStables(t *testing.T) {
}
}
func TestSchemaMigrationAddsTrackSOCIdempotently(t *testing.T) {
statements := strings.Join(SchemaMigrationStatements("test_ts"), "\n")
if !strings.Contains(statements, "ALTER STABLE test_ts.vehicle_locations ADD COLUMN soc_percent DOUBLE") {
t.Fatalf("SOC migration missing: %s", statements)
}
for _, message := range []string{"duplicate column name", "Duplicated column names", "column already exists"} {
if !isDuplicateTDengineColumnError(errors.New(message)) {
t.Fatalf("duplicate TDengine column error should be idempotent: %s", message)
}
}
}
func TestWriterAppendsRawAndLocationOnly(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
@@ -130,6 +144,16 @@ func TestWriterChunksOversizedParsedFields(t *testing.T) {
if got := countSQL(exec.calls, "INSERT INTO chunk_"); got < 2 {
t.Fatalf("chunk insert count = %d, calls=%v", got, exec.calls)
}
chunkInserts := matchingSQL(exec.calls, "INSERT INTO chunk_")
if len(chunkInserts) != 2 {
t.Fatalf("chunk inserts = %d, calls=%v", len(chunkInserts), exec.calls)
}
if !strings.Contains(chunkInserts[0], "VALUES (1782745114999, '") {
t.Fatalf("first chunk ts should use received_at: %s", chunkInserts[0])
}
if !strings.Contains(chunkInserts[1], "VALUES (1782745115000, '") {
t.Fatalf("second chunk ts should be offset by chunk_index ms: %s", chunkInserts[1])
}
}
func TestWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T) {
@@ -185,11 +209,20 @@ func TestTimeLiteralsUseEpochMilliseconds(t *testing.T) {
}
}
func TestStringLiteralPreservesJSONEscapesForTDengine(t *testing.T) {
value := `{"bits":"{\"abs\":false}"}`
want := `'{"bits":"{\\"abs\\":false}"}'`
if got := literal(value); got != want {
t.Fatalf("literal() = %q, want %q", got, want)
}
}
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.Fields = map[string]any{}
env.ParsedFields = map[string]any{}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
@@ -227,6 +260,86 @@ func TestWriterSkipsLocationWhenVINIsMissing(t *testing.T) {
}
}
func TestLocationStatusClassifiesDerivedLocationEligibility(t *testing.T) {
realtimeWithoutCoordinates := sampleEnvelope()
realtimeWithoutCoordinates.ParsedFields = map[string]any{
"jt808.location.speed_kmh": 30,
}
bareFieldsOnly := realtimeWithoutCoordinates
bareFieldsOnly.ParsedFields = nil
bareFieldsOnly.Fields = map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
}
withoutVIN := sampleEnvelope()
withoutVIN.VIN = ""
nonRealtimeWithCoordinates := sampleEnvelope()
nonRealtimeWithCoordinates.MessageID = "0x0100"
tests := []struct {
name string
env envelope.FrameEnvelope
want string
}{
{name: "ok", env: sampleEnvelope(), want: LocationStatusOK},
{name: "non realtime", env: nonRealtimeWithCoordinates, want: LocationStatusSkippedNonRealtime},
{name: "missing vin", env: withoutVIN, want: LocationStatusSkippedMissingVIN},
{name: "missing coordinates", env: realtimeWithoutCoordinates, want: LocationStatusSkippedMissingCoordinates},
{name: "bare fields are not canonical", env: bareFieldsOnly, want: LocationStatusSkippedMissingCoordinates},
}
for _, test := range tests {
if got := LocationStatus(test.env); got != test.want {
t.Fatalf("%s LocationStatus() = %q, want %q", test.name, got, test.want)
}
}
}
func TestWriterSkipsLocationForNonRealtimeFrame(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.MessageID = "0x0100"
env.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 0 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
t.Fatalf("location insert count = %d", got)
}
}
func TestWriterAppendAllWithResultKeepsRawSuccessWhenLocationFails(t *testing.T) {
locationErr := errors.New("location insert failed")
exec := &recordingExec{errs: []error{nil, nil, nil, nil, locationErr}}
writer := NewWriter(exec)
env := sampleEnvelope()
result, err := writer.AppendAllWithResult(context.Background(), env)
if err != nil {
t.Fatalf("AppendAllWithResult() raw error = %v", err)
}
if !errors.Is(result.LocationError, locationErr) {
t.Fatalf("location error = %v, want %v", result.LocationError, locationErr)
}
if result.RawRows != 1 || result.LocationRows != 0 {
t.Fatalf("result = %+v, want raw row retained and no location rows", result)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location insert attempted count = %d", got)
}
}
func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
@@ -262,6 +375,94 @@ func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
}
}
func TestWriterAppendsBatchAcrossChildTablesWithSingleMultiTableInsert(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
first := sampleEnvelope()
second := sampleEnvelope()
second.Sequence = 2
second.EventID = "second-event"
second.VIN = "LNBVIN00000000002"
second.Phone = "013307795426"
second.EventTimeMS += 1000
second.ReceivedAtMS += 1000
if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if got := countSQL(exec.calls, "USING raw_frames"); got != 2 {
t.Fatalf("raw child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 2 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw multi-table insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location multi-table insert count = %d", got)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if got := strings.Count(rawInsert, "\nVALUES "); got != 2 {
t.Fatalf("raw multi-table VALUES sections = %d, sql=%s", got, rawInsert)
}
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if got := strings.Count(locationInsert, "\nVALUES "); got != 2 {
t.Fatalf("location multi-table VALUES sections = %d, sql=%s", got, locationInsert)
}
}
func TestWriterAppendAllBatchSkipsLocationForNonRealtimeFrames(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
first := sampleEnvelope()
first.MessageID = "0x0100"
first.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}}
second := sampleEnvelope()
second.Sequence = 2
second.EventTimeMS += 1000
second.ReceivedAtMS += 1000
if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw batch insert count = %d", got)
}
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if got := strings.Count(locationInsert, "),(") + 1; got != 1 {
t.Fatalf("location batch row count = %d, sql=%s", got, locationInsert)
}
}
func TestWriterNormalizesFarFutureEventTimeForLocationButKeepsRawEvidence(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC)
futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC)
env.ReceivedAtMS = received.UnixMilli()
env.EventTimeMS = futureEvent.UnixMilli()
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if !strings.Contains(rawInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) {
t.Fatalf("raw insert should keep original event time %d: %s", futureEvent.UnixMilli(), rawInsert)
}
if strings.Contains(locationInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) {
t.Fatalf("location insert should not use far future event time: %s", locationInsert)
}
if !strings.Contains(locationInsert, strconv.FormatInt(received.UnixMilli(), 10)) {
t.Fatalf("location insert should use received time %d: %s", received.UnixMilli(), locationInsert)
}
}
func TestWriterWithDatabaseQualifiesTDengineTables(t *testing.T) {
exec := &recordingExec{}
writer := NewWriterWithDatabase(exec, "vehicle_ts")
@@ -372,14 +573,14 @@ func sampleEnvelope() envelope.FrameEnvelope {
ReceivedAtMS: 1782745114999,
RawHex: "7E0200",
Parsed: map[string]any{"message": "location"},
Fields: map[string]any{
envelope.FieldLongitude: 121.069881,
envelope.FieldLatitude: 30.590151,
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10241.2,
"direction_deg": uint16(79),
"alarm_flag": uint32(0),
"status_flag": uint32(72),
ParsedFields: map[string]any{
"jt808.location.longitude": 121.069881,
"jt808.location.latitude": 30.590151,
"jt808.location.speed_kmh": 23.0,
"jt808.location.total_mileage_km": 10241.2,
"jt808.location.direction_deg": uint16(79),
"jt808.location.alarm_flag": uint32(0),
"jt808.location.status_flag": uint32(72),
},
ParseStatus: envelope.ParseOK,
}
@@ -404,6 +605,16 @@ func findSQL(calls []execCall, pattern string) string {
return ""
}
func matchingSQL(calls []execCall, pattern string) []string {
out := []string{}
for _, call := range calls {
if strings.Contains(call.query, pattern) {
out = append(out, call.query)
}
}
return out
}
func containsSQL(calls []execCall, pattern string) bool {
return findSQL(calls, pattern) != ""
}
@@ -428,10 +639,16 @@ type execCall struct {
type recordingExec struct {
calls []execCall
errs []error
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
if len(e.errs) > 0 {
err := e.errs[0]
e.errs = e.errs[1:]
return nil, err
}
return nil, nil
}