feat: build vehicle data platform and production pipeline

go/vehicle-gateway/internal/history/writer.go

@@ -7,6 +7,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -15,6 +16,7 @@ import (
 
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
 )
 
 type Execer interface {
@@ -27,9 +29,23 @@ type Writer struct {
 	cache    tableCache
 }
 
+type AppendResult struct {
+	RawRows       int
+	LocationRows  int
+	LocationError error
+}
+
+const (
+	LocationStatusOK                        = "ok"
+	LocationStatusSkippedNonRealtime        = "skipped_non_realtime"
+	LocationStatusSkippedMissingVIN         = "skipped_missing_vin"
+	LocationStatusSkippedMissingCoordinates = "skipped_missing_coordinates"
+)
+
 const (
 	rawFramePayloadInlineLimit = 12_000
 	rawFramePayloadChunkSize   = 16_000
+	tdengineInsertSoftLimit    = 6 * 1024 * 1024
 )
 
 type payloadChunk struct {
@@ -100,9 +116,22 @@ func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
 			return err
 		}
 	}
+	for _, statement := range SchemaMigrationStatements(database) {
+		if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateTDengineColumnError(err) {
+			return err
+		}
+	}
 	return nil
 }
 
+func isDuplicateTDengineColumnError(err error) bool {
+	if err == nil {
+		return false
+	}
+	message := strings.ToLower(err.Error())
+	return strings.Contains(message, "duplicate column") || strings.Contains(message, "duplicated column") || strings.Contains(message, "column already exists")
+}
+
 func (w *Writer) qualify(table string) string {
 	table = normalizeIdentifier(table)
 	if w.database == "" {
@@ -112,20 +141,52 @@ func (w *Writer) qualify(table string) string {
 }
 
 func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
-	if err := w.AppendRawFrame(ctx, env); err != nil {
+	result, err := w.AppendAllWithResult(ctx, env)
+	if err != nil {
 		return err
 	}
-	return w.AppendLocation(ctx, env)
+	return result.LocationError
+}
+
+func (w *Writer) AppendAllWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) {
+	var result AppendResult
+	if err := w.AppendRawFrame(ctx, env); err != nil {
+		return result, err
+	}
+	result.RawRows = 1
+	rows, err := w.appendLocationWithCount(ctx, env)
+	if err != nil {
+		result.LocationError = err
+		return result, nil
+	}
+	result.LocationRows = rows
+	return result, nil
 }
 
 func (w *Writer) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
-	if len(envelopes) == 0 {
-		return nil
-	}
-	if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
+	result, err := w.AppendAllBatchWithResult(ctx, envelopes)
+	if err != nil {
 		return err
 	}
-	return w.AppendLocationBatch(ctx, envelopes)
+	return result.LocationError
+}
+
+func (w *Writer) AppendAllBatchWithResult(ctx context.Context, envelopes []envelope.FrameEnvelope) (AppendResult, error) {
+	var result AppendResult
+	if len(envelopes) == 0 {
+		return result, nil
+	}
+	if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
+		return result, err
+	}
+	result.RawRows = len(envelopes)
+	rows, err := w.appendLocationBatchWithCount(ctx, envelopes)
+	if err != nil {
+		result.LocationError = err
+		return result, nil
+	}
+	result.LocationRows = rows
+	return result, nil
 }
 
 func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -165,7 +226,6 @@ VALUES (%s)`, w.qualify(chunkTable), joinLiterals(chunkValues(env, chunk)))); er
 func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
 	rowsByTable := map[string][]string{}
 	chunkRowsByTable := map[string][]string{}
-	chunkEnvByTable := map[string]envelope.FrameEnvelope{}
 	for _, env := range envelopes {
 		table := tableName("raw", env)
 		if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
@@ -184,87 +244,141 @@ func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.F
 		if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
 			return err
 		}
-		chunkEnvByTable[chunkTable] = env
 		for _, chunk := range chunks {
 			chunkRowsByTable[chunkTable] = append(chunkRowsByTable[chunkTable], "("+joinLiterals(chunkValues(env, chunk))+")")
 		}
 	}
-	for table, rows := range rowsByTable {
-		if len(rows) == 0 {
-			continue
-		}
-		if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
-  (ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
-   raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
-VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
-			return err
-		}
+	if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
+   raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint`); err != nil {
+		return err
 	}
-	for table, rows := range chunkRowsByTable {
-		if len(rows) == 0 {
-			continue
-		}
-		_ = chunkEnvByTable[table]
-		if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
-  (ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
-VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
-			return err
-		}
+	if err := w.execMultiTableInsert(ctx, chunkRowsByTable, `ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text`); err != nil {
+		return err
 	}
 	return nil
 }
 
 func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
-	if strings.TrimSpace(env.VIN) == "" {
-		return nil
-	}
-	longitude, okLon := floatField(env, envelope.FieldLongitude)
-	latitude, okLat := floatField(env, envelope.FieldLatitude)
-	if !okLon || !okLat {
-		return nil
-	}
-	table := locationTableName(env)
-	if err := w.ensureLocationChild(ctx, table, env); err != nil {
-		return err
-	}
-	_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
-  (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
-   direction_deg, alarm_flag, status_flag, total_mileage_km)
-VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
+	_, err := w.appendLocationWithCount(ctx, env)
 	return err
 }
 
+func (w *Writer) appendLocationWithCount(ctx context.Context, env envelope.FrameEnvelope) (int, error) {
+	longitude, latitude, status := locationCandidate(env)
+	if status != LocationStatusOK {
+		return 0, nil
+	}
+	table := locationTableName(env)
+	if err := w.ensureLocationChild(ctx, table, env); err != nil {
+		return 0, err
+	}
+	_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
+  (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
+   soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km)
+VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
+	if err != nil {
+		return 0, err
+	}
+	return 1, nil
+}
+
 func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
+	_, err := w.appendLocationBatchWithCount(ctx, envelopes)
+	return err
+}
+
+func (w *Writer) appendLocationBatchWithCount(ctx context.Context, envelopes []envelope.FrameEnvelope) (int, error) {
 	rowsByTable := map[string][]string{}
+	rowCount := 0
 	for _, env := range envelopes {
-		if strings.TrimSpace(env.VIN) == "" {
-			continue
-		}
-		longitude, okLon := floatField(env, envelope.FieldLongitude)
-		latitude, okLat := floatField(env, envelope.FieldLatitude)
-		if !okLon || !okLat {
+		longitude, latitude, status := locationCandidate(env)
+		if status != LocationStatusOK {
 			continue
 		}
 		table := locationTableName(env)
 		if err := w.ensureLocationChild(ctx, table, env); err != nil {
-			return err
+			return rowCount, err
 		}
 		rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(locationValues(env, longitude, latitude))+")")
+		rowCount++
 	}
-	for table, rows := range rowsByTable {
-		if len(rows) == 0 {
-			continue
-		}
-		if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
-  (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
-   direction_deg, alarm_flag, status_flag, total_mileage_km)
-VALUES %s`, w.qualify(table), strings.Join(rows, ","))); err != nil {
+	if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
+   soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km`); err != nil {
+		return rowCount, err
+	}
+	return rowCount, nil
+}
+
+func (w *Writer) execMultiTableInsert(ctx context.Context, rowsByTable map[string][]string, columns string) error {
+	statements := buildMultiTableInsertStatements(rowsByTable, w.qualify, columns, tdengineInsertSoftLimit)
+	for _, statement := range statements {
+		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
+func buildMultiTableInsertStatements(rowsByTable map[string][]string, qualify func(string) string, columns string, softLimit int) []string {
+	if len(rowsByTable) == 0 {
+		return nil
+	}
+	keys := make([]string, 0, len(rowsByTable))
+	for table, rows := range rowsByTable {
+		if len(rows) > 0 {
+			keys = append(keys, table)
+		}
+	}
+	sort.Strings(keys)
+	if len(keys) == 0 {
+		return nil
+	}
+	if qualify == nil {
+		qualify = func(table string) string { return table }
+	}
+	var statements []string
+	current := "INSERT INTO "
+	parts := 0
+	for _, table := range keys {
+		part := fmt.Sprintf(`%s
+  (%s)
+VALUES %s`, qualify(table), columns, strings.Join(rowsByTable[table], ","))
+		if parts > 0 && softLimit > 0 && len(current)+1+len(part) > softLimit {
+			statements = append(statements, current)
+			current = "INSERT INTO "
+			parts = 0
+		}
+		if parts > 0 {
+			current += " "
+		}
+		current += part
+		parts++
+	}
+	if parts > 0 {
+		statements = append(statements, current)
+	}
+	return statements
+}
+
+func LocationStatus(env envelope.FrameEnvelope) string {
+	_, _, status := locationCandidate(env)
+	return status
+}
+
+func locationCandidate(env envelope.FrameEnvelope) (float64, float64, string) {
+	if !envelope.IsRealtimeTelemetryFrame(env) {
+		return 0, 0, LocationStatusSkippedNonRealtime
+	}
+	if strings.TrimSpace(env.VIN) == "" {
+		return 0, 0, LocationStatusSkippedMissingVIN
+	}
+	location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
+	if !ok {
+		return 0, 0, LocationStatusSkippedMissingCoordinates
+	}
+	return location.Longitude, location.Latitude, LocationStatusOK
+}
+
 func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
 	key := stable + "." + table
 	return w.cache.doOnce(key, func() error {
@@ -325,7 +439,7 @@ func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsed
 func chunkValues(env envelope.FrameEnvelope, chunk payloadChunk) []any {
 	received := millis(env.ReceivedAtMS)
 	return []any{
-		received,
+		received.Add(time.Duration(chunk.Index) * time.Millisecond),
 		env.StableEventID(),
 		frameID(env),
 		received,
@@ -379,18 +493,21 @@ func safeChunkEnd(value string, start int, maxBytes int) int {
 
 func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
 	received := millis(env.ReceivedAtMS)
+	location, _ := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
+	totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
 	return []any{
 		eventTimeOrReceived(env),
 		env.StableEventID(),
 		received,
 		longitude,
 		latitude,
-		floatFieldOrNil(env, "altitude_m"),
-		floatFieldOrNil(env, envelope.FieldSpeedKMH),
-		intFieldOrNil(env, "direction_deg"),
-		intFieldOrNil(env, "alarm_flag"),
-		intFieldOrNil(env, "status_flag"),
-		floatFieldOrNil(env, envelope.FieldTotalMileageKM),
+		optionalFloat(location.AltitudeM),
+		optionalFloat(location.SpeedKMH),
+		optionalFloat(location.SOCPercent),
+		optionalInt(location.DirectionDeg),
+		optionalInt64(location.AlarmFlag),
+		optionalInt64(location.StatusFlag),
+		optionalPositiveFloat(totalMileageKM, hasTotalMileage),
 	}
 }
 
@@ -463,76 +580,37 @@ func parsedFieldsJSONString(env envelope.FrameEnvelope) string {
 	return jsonString(fields)
 }
 
-func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
-	if env.Fields == nil {
-		return 0, false
-	}
-	value, ok := env.Fields[key]
-	if !ok || value == nil {
-		return 0, false
-	}
-	switch typed := value.(type) {
-	case float64:
-		return typed, true
-	case float32:
-		return float64(typed), true
-	case int:
-		return float64(typed), true
-	case int64:
-		return float64(typed), true
-	case uint16:
-		return float64(typed), true
-	case uint32:
-		return float64(typed), true
-	case string:
-		parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64)
-		return parsed, err == nil
-	default:
-		return 0, false
+func optionalFloat(value *float64) any {
+	if value == nil {
+		return nil
 	}
+	return *value
 }
 
-func floatFieldOrNil(env envelope.FrameEnvelope, key string) any {
-	value, ok := floatField(env, key)
-	if !ok {
+func optionalInt(value *float64) any {
+	if value == nil {
+		return nil
+	}
+	return int64(*value)
+}
+
+func optionalInt64(value *int64) any {
+	if value == nil {
+		return nil
+	}
+	return *value
+}
+
+func optionalPositiveFloat(value float64, ok bool) any {
+	if !ok || value <= 0 {
 		return nil
 	}
 	return value
 }
 
-func intFieldOrNil(env envelope.FrameEnvelope, key string) any {
-	if env.Fields == nil {
-		return nil
-	}
-	value, ok := env.Fields[key]
-	if !ok || value == nil {
-		return nil
-	}
-	switch typed := value.(type) {
-	case int:
-		return typed
-	case int64:
-		return typed
-	case uint16:
-		return int64(typed)
-	case uint32:
-		return int64(typed)
-	case float64:
-		return int64(typed)
-	case string:
-		parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
-		if err == nil {
-			return parsed
-		}
-	}
-	return nil
-}
-
 func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
-	if env.EventTimeMS > 0 {
-		return millis(env.EventTimeMS)
-	}
-	return millis(env.ReceivedAtMS)
+	eventMS, _ := envelope.NormalizedEventTimeMS(env)
+	return millis(eventMS)
 }
 
 func millis(value int64) time.Time {
@@ -543,7 +621,10 @@ func millis(value int64) time.Time {
 }
 
 func quote(value string) string {
-	return strings.ReplaceAll(value, "'", "''")
+	return strings.NewReplacer(
+		`\`, `\\`,
+		`'`, `''`,
+	).Replace(value)
 }
 
 func joinLiterals(values []any) string {