shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
go/vehicle-gateway/internal/history/query.go
View File

@@ -76,6 +76,7 @@ type LocationRow struct {
Latitude float64 `json:"latitude"`
AltitudeM *float64 `json:"altitude_m,omitempty"`
SpeedKMH *float64 `json:"speed_kmh,omitempty"`
SOCPercent *float64 `json:"soc_percent,omitempty"`
DirectionDeg *int64 `json:"direction_deg,omitempty"`
AlarmFlag *int64 `json:"alarm_flag,omitempty"`
StatusFlag *int64 `json:"status_flag,omitempty"`
@@ -218,6 +219,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]
var receivedAt scanDateTime
var altitude sql.NullFloat64
var speed sql.NullFloat64
var soc sql.NullFloat64
var direction sql.NullInt64
var alarm sql.NullInt64
var status sql.NullInt64
@@ -230,6 +232,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]
&row.Latitude,
&altitude,
&speed,
&soc,
&direction,
&alarm,
&status,
@@ -243,6 +246,7 @@ func (r *LocationRepository) Query(ctx context.Context, query LocationQuery) ([]
row.ReceivedAt = receivedAt.String
row.AltitudeM = nullableFloat(altitude)
row.SpeedKMH = nullableFloat(speed)
row.SOCPercent = nullableFloat(soc)
row.DirectionDeg = nullableInt(direction)
row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status)
@@ -563,7 +567,7 @@ func quotedList(values []string) string {
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vin FROM ` + table
sqlText := `SELECT ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh, soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vin FROM ` + table
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
@@ -954,11 +958,11 @@ func normalizeDateTimeLiteral(value string) string {
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
return parsed.In(shanghai).Format("2006-01-02 15:04:05")
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
return parsed.In(shanghai).Format("2006-01-02 15:04:05")
}
return value
}

28
go/vehicle-gateway/internal/history/query_test.go
View File

@@ -391,10 +391,10 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
"soc_percent", "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
121.07764, 30.585928, 11.0, 8.0, 82.5, 171, 0, 4718595, 8792.8,
"JT808", "LKLG7C4E3NA774736",
))
@@ -408,7 +408,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vin":"LKLG7C4E3NA774736"`, `"longitude":121.07764`, `"total_mileage_km":8792.8`, `"total":17`} {
for _, want := range []string{`"vin":"LKLG7C4E3NA774736"`, `"longitude":121.07764`, `"soc_percent":82.5`, `"total_mileage_km":8792.8`, `"total":17`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
@@ -432,10 +432,10 @@ func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) {
mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
"soc_percent", "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
121.07764, 30.585928, 11.0, 8.0, 82.5, 171, 0, 4718595, 8792.8,
"JT808", "LKLG7C4E3NA774736",
))
@@ -476,17 +476,17 @@ func TestParseRawFrameQueryAcceptsDatetimeLocalValues(t *testing.T) {
if err != nil {
t.Fatalf("parseRawFrameQuery() error = %v", err)
}
if query.DateFrom != "2026-06-30 16:00:00" || query.DateTo != "2026-07-01 16:00:00" {
if query.DateFrom != "2026-07-01 00:00:00" || query.DateTo != "2026-07-02 00:00:00" {
t.Fatalf("date range = %q -> %q", query.DateFrom, query.DateTo)
}
}
func TestNormalizeDateTimeLiteralConvertsInputToTDengineUTCTime(t *testing.T) {
func TestNormalizeDateTimeLiteralUsesAsiaShanghaiQueryTime(t *testing.T) {
for raw, want := range map[string]string{
"2026-07-01T00:00:00": "2026-06-30 16:00:00",
"2026-07-01 00:00:00": "2026-06-30 16:00:00",
"2026-07-01T00:00:00+08:00": "2026-06-30 16:00:00",
"2026-06-30T16:00:00Z": "2026-06-30 16:00:00",
"2026-07-01T00:00:00": "2026-07-01 00:00:00",
"2026-07-01 00:00:00": "2026-07-01 00:00:00",
"2026-07-01T00:00:00+08:00": "2026-07-01 00:00:00",
"2026-06-30T16:00:00Z": "2026-07-01 00:00:00",
} {
if got := normalizeDateTimeLiteral(raw); got != want {
t.Fatalf("normalizeDateTimeLiteral(%q) = %q, want %q", raw, got, want)
@@ -522,7 +522,7 @@ func TestBuildLocationSQLUsesLiteralsForTDengine(t *testing.T) {
for _, want := range []string{
"protocol = 'JT808'",
"vin = 'LKLG7C4E3NA774736'",
"ts >= '2026-07-01 16:00:00'",
"ts >= '2026-07-02 00:00:00'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {
@@ -556,8 +556,8 @@ func TestBuildRawFrameSQLUsesLiteralsForTDengine(t *testing.T) {
"vehicle_key = 'JT808:013307811350'",
"vin = 'VIN''1'",
"message_id = 512",
"ts >= '2026-06-30 16:00:00'",
"ts <= '2026-07-01 15:59:59'",
"ts >= '2026-07-01 00:00:00'",
"ts <= '2026-07-01 23:59:59'",
"LIMIT 20 OFFSET 5",
} {
if !strings.Contains(sqlText, want) {

11
go/vehicle-gateway/internal/history/schema.go
View File

@@ -54,6 +54,7 @@ func SchemaStatements(database string) []string {
latitude DOUBLE,
altitude_m DOUBLE,
speed_kmh DOUBLE,
soc_percent DOUBLE,
direction_deg INT,
alarm_flag BIGINT,
status_flag BIGINT,
@@ -64,3 +65,13 @@ func SchemaStatements(database string) []string {
)`,
}
}
// SchemaMigrationStatements contains additive TDengine changes that must also
// be applied to an already existing stable. The writer treats duplicate-column
// errors as success so startup remains idempotent across releases.
func SchemaMigrationStatements(database string) []string {
if database == "" {
database = DefaultDatabase
}
return []string{"ALTER STABLE " + database + ".vehicle_locations ADD COLUMN soc_percent DOUBLE"}
}

339
go/vehicle-gateway/internal/history/writer.go
View File

@@ -7,6 +7,7 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"sync"
@@ -15,6 +16,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)
type Execer interface {
@@ -27,9 +29,23 @@ type Writer struct {
cache tableCache
}
type AppendResult struct {
RawRows int
LocationRows int
LocationError error
}
const (
LocationStatusOK = "ok"
LocationStatusSkippedNonRealtime = "skipped_non_realtime"
LocationStatusSkippedMissingVIN = "skipped_missing_vin"
LocationStatusSkippedMissingCoordinates = "skipped_missing_coordinates"
)
const (
rawFramePayloadInlineLimit = 12_000
rawFramePayloadChunkSize = 16_000
tdengineInsertSoftLimit = 6 * 1024 * 1024
)
type payloadChunk struct {
@@ -100,9 +116,22 @@ func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
return err
}
}
for _, statement := range SchemaMigrationStatements(database) {
if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateTDengineColumnError(err) {
return err
}
}
return nil
}
func isDuplicateTDengineColumnError(err error) bool {
if err == nil {
return false
}
message := strings.ToLower(err.Error())
return strings.Contains(message, "duplicate column") || strings.Contains(message, "duplicated column") || strings.Contains(message, "column already exists")
}
func (w *Writer) qualify(table string) string {
table = normalizeIdentifier(table)
if w.database == "" {
@@ -112,20 +141,52 @@ func (w *Writer) qualify(table string) string {
}
func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
if err := w.AppendRawFrame(ctx, env); err != nil {
result, err := w.AppendAllWithResult(ctx, env)
if err != nil {
return err
}
return w.AppendLocation(ctx, env)
return result.LocationError
}
func (w *Writer) AppendAllWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) {
var result AppendResult
if err := w.AppendRawFrame(ctx, env); err != nil {
return result, err
}
result.RawRows = 1
rows, err := w.appendLocationWithCount(ctx, env)
if err != nil {
result.LocationError = err
return result, nil
}
result.LocationRows = rows
return result, nil
}
func (w *Writer) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
if len(envelopes) == 0 {
return nil
}
if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
result, err := w.AppendAllBatchWithResult(ctx, envelopes)
if err != nil {
return err
}
return w.AppendLocationBatch(ctx, envelopes)
return result.LocationError
}
func (w *Writer) AppendAllBatchWithResult(ctx context.Context, envelopes []envelope.FrameEnvelope) (AppendResult, error) {
var result AppendResult
if len(envelopes) == 0 {
return result, nil
}
if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
return result, err
}
result.RawRows = len(envelopes)
rows, err := w.appendLocationBatchWithCount(ctx, envelopes)
if err != nil {
result.LocationError = err
return result, nil
}
result.LocationRows = rows
return result, nil
}
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -165,7 +226,6 @@ VALUES (%s)`, w.qualify(chunkTable), joinLiterals(chunkValues(env, chunk)))); er
func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
rowsByTable := map[string][]string{}
chunkRowsByTable := map[string][]string{}
chunkEnvByTable := map[string]envelope.FrameEnvelope{}
for _, env := range envelopes {
table := tableName("raw", env)
if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
@@ -184,87 +244,141 @@ func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.F
if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
return err
}
chunkEnvByTable[chunkTable] = env
for _, chunk := range chunks {
chunkRowsByTable[chunkTable] = append(chunkRowsByTable[chunkTable], "("+joinLiterals(chunkValues(env, chunk))+")")
}
}
for table, rows := range rowsByTable {
if len(rows) == 0 {
continue
}
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
return err
}
if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint`); err != nil {
return err
}
for table, rows := range chunkRowsByTable {
if len(rows) == 0 {
continue
}
_ = chunkEnvByTable[table]
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
return err
}
if err := w.execMultiTableInsert(ctx, chunkRowsByTable, `ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text`); err != nil {
return err
}
return nil
}
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
if strings.TrimSpace(env.VIN) == "" {
return nil
}
longitude, okLon := floatField(env, envelope.FieldLongitude)
latitude, okLat := floatField(env, envelope.FieldLatitude)
if !okLon || !okLat {
return nil
}
table := locationTableName(env)
if err := w.ensureLocationChild(ctx, table, env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
_, err := w.appendLocationWithCount(ctx, env)
return err
}
func (w *Writer) appendLocationWithCount(ctx context.Context, env envelope.FrameEnvelope) (int, error) {
longitude, latitude, status := locationCandidate(env)
if status != LocationStatusOK {
return 0, nil
}
table := locationTableName(env)
if err := w.ensureLocationChild(ctx, table, env); err != nil {
return 0, err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
if err != nil {
return 0, err
}
return 1, nil
}
func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
_, err := w.appendLocationBatchWithCount(ctx, envelopes)
return err
}
func (w *Writer) appendLocationBatchWithCount(ctx context.Context, envelopes []envelope.FrameEnvelope) (int, error) {
rowsByTable := map[string][]string{}
rowCount := 0
for _, env := range envelopes {
if strings.TrimSpace(env.VIN) == "" {
continue
}
longitude, okLon := floatField(env, envelope.FieldLongitude)
latitude, okLat := floatField(env, envelope.FieldLatitude)
if !okLon || !okLat {
longitude, latitude, status := locationCandidate(env)
if status != LocationStatusOK {
continue
}
table := locationTableName(env)
if err := w.ensureLocationChild(ctx, table, env); err != nil {
return err
return rowCount, err
}
rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(locationValues(env, longitude, latitude))+")")
rowCount++
}
for table, rows := range rowsByTable {
if len(rows) == 0 {
continue
}
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km`); err != nil {
return rowCount, err
}
return rowCount, nil
}
func (w *Writer) execMultiTableInsert(ctx context.Context, rowsByTable map[string][]string, columns string) error {
statements := buildMultiTableInsertStatements(rowsByTable, w.qualify, columns, tdengineInsertSoftLimit)
for _, statement := range statements {
if _, err := w.exec.ExecContext(ctx, statement); err != nil {
return err
}
}
return nil
}
func buildMultiTableInsertStatements(rowsByTable map[string][]string, qualify func(string) string, columns string, softLimit int) []string {
if len(rowsByTable) == 0 {
return nil
}
keys := make([]string, 0, len(rowsByTable))
for table, rows := range rowsByTable {
if len(rows) > 0 {
keys = append(keys, table)
}
}
sort.Strings(keys)
if len(keys) == 0 {
return nil
}
if qualify == nil {
qualify = func(table string) string { return table }
}
var statements []string
current := "INSERT INTO "
parts := 0
for _, table := range keys {
part := fmt.Sprintf(`%s
(%s)
VALUES %s`, qualify(table), columns, strings.Join(rowsByTable[table], ","))
if parts > 0 && softLimit > 0 && len(current)+1+len(part) > softLimit {
statements = append(statements, current)
current = "INSERT INTO "
parts = 0
}
if parts > 0 {
current += " "
}
current += part
parts++
}
if parts > 0 {
statements = append(statements, current)
}
return statements
}
func LocationStatus(env envelope.FrameEnvelope) string {
_, _, status := locationCandidate(env)
return status
}
func locationCandidate(env envelope.FrameEnvelope) (float64, float64, string) {
if !envelope.IsRealtimeTelemetryFrame(env) {
return 0, 0, LocationStatusSkippedNonRealtime
}
if strings.TrimSpace(env.VIN) == "" {
return 0, 0, LocationStatusSkippedMissingVIN
}
location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
if !ok {
return 0, 0, LocationStatusSkippedMissingCoordinates
}
return location.Longitude, location.Latitude, LocationStatusOK
}
func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
key := stable + "." + table
return w.cache.doOnce(key, func() error {
@@ -325,7 +439,7 @@ func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsed
func chunkValues(env envelope.FrameEnvelope, chunk payloadChunk) []any {
received := millis(env.ReceivedAtMS)
return []any{
received,
received.Add(time.Duration(chunk.Index) * time.Millisecond),
env.StableEventID(),
frameID(env),
received,
@@ -379,18 +493,21 @@ func safeChunkEnd(value string, start int, maxBytes int) int {
func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
received := millis(env.ReceivedAtMS)
location, _ := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
received,
longitude,
latitude,
floatFieldOrNil(env, "altitude_m"),
floatFieldOrNil(env, envelope.FieldSpeedKMH),
intFieldOrNil(env, "direction_deg"),
intFieldOrNil(env, "alarm_flag"),
intFieldOrNil(env, "status_flag"),
floatFieldOrNil(env, envelope.FieldTotalMileageKM),
optionalFloat(location.AltitudeM),
optionalFloat(location.SpeedKMH),
optionalFloat(location.SOCPercent),
optionalInt(location.DirectionDeg),
optionalInt64(location.AlarmFlag),
optionalInt64(location.StatusFlag),
optionalPositiveFloat(totalMileageKM, hasTotalMileage),
}
}
@@ -463,76 +580,37 @@ func parsedFieldsJSONString(env envelope.FrameEnvelope) string {
return jsonString(fields)
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil {
return 0, false
}
value, ok := env.Fields[key]
if !ok || value == nil {
return 0, false
}
switch typed := value.(type) {
case float64:
return typed, true
case float32:
return float64(typed), true
case int:
return float64(typed), true
case int64:
return float64(typed), true
case uint16:
return float64(typed), true
case uint32:
return float64(typed), true
case string:
parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
return parsed, err == nil
default:
return 0, false
func optionalFloat(value *float64) any {
if value == nil {
return nil
}
return *value
}
func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
value, ok := floatField(env, key)
if !ok {
func optionalInt(value *float64) any {
if value == nil {
return nil
}
return int64(*value)
}
func optionalInt64(value *int64) any {
if value == nil {
return nil
}
return *value
}
func optionalPositiveFloat(value float64, ok bool) any {
if !ok || value <= 0 {
return nil
}
return value
}
func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
if env.Fields == nil {
return nil
}
value, ok := env.Fields[key]
if !ok || value == nil {
return nil
}
switch typed := value.(type) {
case int:
return typed
case int64:
return typed
case uint16:
return int64(typed)
case uint32:
return int64(typed)
case float64:
return int64(typed)
case string:
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
if err == nil {
return parsed
}
}
return nil
}
func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
if env.EventTimeMS > 0 {
return millis(env.EventTimeMS)
}
return millis(env.ReceivedAtMS)
eventMS, _ := envelope.NormalizedEventTimeMS(env)
return millis(eventMS)
}
func millis(value int64) time.Time {
@@ -543,7 +621,10 @@ func millis(value int64) time.Time {
}
func quote(value string) string {
return strings.ReplaceAll(value, "'", "''")
return strings.NewReplacer(
`\`, `\\`,
`'`, `''`,
).Replace(value)
}
func joinLiterals(values []any) string {

235
go/vehicle-gateway/internal/history/writer_test.go
View File

@@ -3,6 +3,8 @@ package history
import (
"context"
"database/sql"
"errors"
"strconv"
"strings"
"sync"
"testing"
@@ -32,6 +34,18 @@ func TestSchemaStatementsCreateCoreStables(t *testing.T) {
}
}
func TestSchemaMigrationAddsTrackSOCIdempotently(t *testing.T) {
statements := strings.Join(SchemaMigrationStatements("test_ts"), "\n")
if !strings.Contains(statements, "ALTER STABLE test_ts.vehicle_locations ADD COLUMN soc_percent DOUBLE") {
t.Fatalf("SOC migration missing: %s", statements)
}
for _, message := range []string{"duplicate column name", "Duplicated column names", "column already exists"} {
if !isDuplicateTDengineColumnError(errors.New(message)) {
t.Fatalf("duplicate TDengine column error should be idempotent: %s", message)
}
}
}
func TestWriterAppendsRawAndLocationOnly(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
@@ -130,6 +144,16 @@ func TestWriterChunksOversizedParsedFields(t *testing.T) {
if got := countSQL(exec.calls, "INSERT INTO chunk_"); got < 2 {
t.Fatalf("chunk insert count = %d, calls=%v", got, exec.calls)
}
chunkInserts := matchingSQL(exec.calls, "INSERT INTO chunk_")
if len(chunkInserts) != 2 {
t.Fatalf("chunk inserts = %d, calls=%v", len(chunkInserts), exec.calls)
}
if !strings.Contains(chunkInserts[0], "VALUES (1782745114999, '") {
t.Fatalf("first chunk ts should use received_at: %s", chunkInserts[0])
}
if !strings.Contains(chunkInserts[1], "VALUES (1782745115000, '") {
t.Fatalf("second chunk ts should be offset by chunk_index ms: %s", chunkInserts[1])
}
}
func TestWriterUsesEnvelopeParsedFieldsWithoutReflattening(t *testing.T) {
@@ -185,11 +209,20 @@ func TestTimeLiteralsUseEpochMilliseconds(t *testing.T) {
}
}
func TestStringLiteralPreservesJSONEscapesForTDengine(t *testing.T) {
value := `{"bits":"{\"abs\":false}"}`
want := `'{"bits":"{\\"abs\\":false}"}'`
if got := literal(value); got != want {
t.Fatalf("literal() = %q, want %q", got, want)
}
}
func TestWriterSkipsSparseDerivedRows(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.Fields = map[string]any{}
env.ParsedFields = map[string]any{}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
@@ -227,6 +260,86 @@ func TestWriterSkipsLocationWhenVINIsMissing(t *testing.T) {
}
}
func TestLocationStatusClassifiesDerivedLocationEligibility(t *testing.T) {
realtimeWithoutCoordinates := sampleEnvelope()
realtimeWithoutCoordinates.ParsedFields = map[string]any{
"jt808.location.speed_kmh": 30,
}
bareFieldsOnly := realtimeWithoutCoordinates
bareFieldsOnly.ParsedFields = nil
bareFieldsOnly.Fields = map[string]any{
envelope.FieldLatitude: 30.590151,
envelope.FieldLongitude: 121.069881,
}
withoutVIN := sampleEnvelope()
withoutVIN.VIN = ""
nonRealtimeWithCoordinates := sampleEnvelope()
nonRealtimeWithCoordinates.MessageID = "0x0100"
tests := []struct {
name string
env envelope.FrameEnvelope
want string
}{
{name: "ok", env: sampleEnvelope(), want: LocationStatusOK},
{name: "non realtime", env: nonRealtimeWithCoordinates, want: LocationStatusSkippedNonRealtime},
{name: "missing vin", env: withoutVIN, want: LocationStatusSkippedMissingVIN},
{name: "missing coordinates", env: realtimeWithoutCoordinates, want: LocationStatusSkippedMissingCoordinates},
{name: "bare fields are not canonical", env: bareFieldsOnly, want: LocationStatusSkippedMissingCoordinates},
}
for _, test := range tests {
if got := LocationStatus(test.env); got != test.want {
t.Fatalf("%s LocationStatus() = %q, want %q", test.name, got, test.want)
}
}
}
func TestWriterSkipsLocationForNonRealtimeFrame(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
env.MessageID = "0x0100"
env.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}}
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 0 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 0 {
t.Fatalf("location insert count = %d", got)
}
}
func TestWriterAppendAllWithResultKeepsRawSuccessWhenLocationFails(t *testing.T) {
locationErr := errors.New("location insert failed")
exec := &recordingExec{errs: []error{nil, nil, nil, nil, locationErr}}
writer := NewWriter(exec)
env := sampleEnvelope()
result, err := writer.AppendAllWithResult(context.Background(), env)
if err != nil {
t.Fatalf("AppendAllWithResult() raw error = %v", err)
}
if !errors.Is(result.LocationError, locationErr) {
t.Fatalf("location error = %v, want %v", result.LocationError, locationErr)
}
if result.RawRows != 1 || result.LocationRows != 0 {
t.Fatalf("result = %+v, want raw row retained and no location rows", result)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location insert attempted count = %d", got)
}
}
func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
@@ -262,6 +375,94 @@ func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
}
}
func TestWriterAppendsBatchAcrossChildTablesWithSingleMultiTableInsert(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
first := sampleEnvelope()
second := sampleEnvelope()
second.Sequence = 2
second.EventID = "second-event"
second.VIN = "LNBVIN00000000002"
second.Phone = "013307795426"
second.EventTimeMS += 1000
second.ReceivedAtMS += 1000
if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if got := countSQL(exec.calls, "USING raw_frames"); got != 2 {
t.Fatalf("raw child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 2 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw multi-table insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location multi-table insert count = %d", got)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if got := strings.Count(rawInsert, "\nVALUES "); got != 2 {
t.Fatalf("raw multi-table VALUES sections = %d, sql=%s", got, rawInsert)
}
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if got := strings.Count(locationInsert, "\nVALUES "); got != 2 {
t.Fatalf("location multi-table VALUES sections = %d, sql=%s", got, locationInsert)
}
}
func TestWriterAppendAllBatchSkipsLocationForNonRealtimeFrames(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
first := sampleEnvelope()
first.MessageID = "0x0100"
first.Parsed = map[string]any{"registration": map[string]any{"plate": "沪A12345"}}
second := sampleEnvelope()
second.Sequence = 2
second.EventTimeMS += 1000
second.ReceivedAtMS += 1000
if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw batch insert count = %d", got)
}
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if got := strings.Count(locationInsert, "),(") + 1; got != 1 {
t.Fatalf("location batch row count = %d, sql=%s", got, locationInsert)
}
}
func TestWriterNormalizesFarFutureEventTimeForLocationButKeepsRawEvidence(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
env := sampleEnvelope()
received := time.Date(2026, 7, 12, 9, 30, 0, 0, time.UTC)
futureEvent := time.Date(2026, 7, 13, 9, 30, 0, 0, time.UTC)
env.ReceivedAtMS = received.UnixMilli()
env.EventTimeMS = futureEvent.UnixMilli()
if err := writer.AppendAll(context.Background(), env); err != nil {
t.Fatalf("AppendAll() error = %v", err)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if !strings.Contains(rawInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) {
t.Fatalf("raw insert should keep original event time %d: %s", futureEvent.UnixMilli(), rawInsert)
}
if strings.Contains(locationInsert, strconv.FormatInt(futureEvent.UnixMilli(), 10)) {
t.Fatalf("location insert should not use far future event time: %s", locationInsert)
}
if !strings.Contains(locationInsert, strconv.FormatInt(received.UnixMilli(), 10)) {
t.Fatalf("location insert should use received time %d: %s", received.UnixMilli(), locationInsert)
}
}
func TestWriterWithDatabaseQualifiesTDengineTables(t *testing.T) {
exec := &recordingExec{}
writer := NewWriterWithDatabase(exec, "vehicle_ts")
@@ -372,14 +573,14 @@ func sampleEnvelope() envelope.FrameEnvelope {
ReceivedAtMS: 1782745114999,
RawHex: "7E0200",
Parsed: map[string]any{"message": "location"},
Fields: map[string]any{
envelope.FieldLongitude: 121.069881,
envelope.FieldLatitude: 30.590151,
envelope.FieldSpeedKMH: 23.0,
envelope.FieldTotalMileageKM: 10241.2,
"direction_deg": uint16(79),
"alarm_flag": uint32(0),
"status_flag": uint32(72),
ParsedFields: map[string]any{
"jt808.location.longitude": 121.069881,
"jt808.location.latitude": 30.590151,
"jt808.location.speed_kmh": 23.0,
"jt808.location.total_mileage_km": 10241.2,
"jt808.location.direction_deg": uint16(79),
"jt808.location.alarm_flag": uint32(0),
"jt808.location.status_flag": uint32(72),
},
ParseStatus: envelope.ParseOK,
}
@@ -404,6 +605,16 @@ func findSQL(calls []execCall, pattern string) string {
return ""
}
func matchingSQL(calls []execCall, pattern string) []string {
out := []string{}
for _, call := range calls {
if strings.Contains(call.query, pattern) {
out = append(out, call.query)
}
}
return out
}
func containsSQL(calls []execCall, pattern string) bool {
return findSQL(calls, pattern) != ""
}
@@ -428,10 +639,16 @@ type execCall struct {
type recordingExec struct {
calls []execCall
errs []error
}
func (e *recordingExec) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) {
e.calls = append(e.calls, execCall{query: query, args: args})
if len(e.errs) > 0 {
err := e.errs[0]
e.errs = e.errs[1:]
return nil, err
}
return nil, nil
}