lingniu-vehicle-ingest/go/vehicle-gateway/internal/history/writer.go

package history

import (
	"context"
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/telemetry"
)

type Execer interface {
	ExecContext(context.Context, string, ...any) (sql.Result, error)
}

type Writer struct {
	exec     Execer
	database string
	cache    tableCache
}

type AppendResult struct {
	RawRows       int
	LocationRows  int
	LocationError error
}

const (
	LocationStatusOK                        = "ok"
	LocationStatusSkippedNonRealtime        = "skipped_non_realtime"
	LocationStatusSkippedMissingVIN         = "skipped_missing_vin"
	LocationStatusSkippedMissingCoordinates = "skipped_missing_coordinates"
)

const (
	rawFramePayloadInlineLimit = 12_000
	rawFramePayloadChunkSize   = 16_000
	tdengineInsertSoftLimit    = 6 * 1024 * 1024
)

type payloadChunk struct {
	Kind  string
	Index int
	Count int
	Text  string
}

type tableCache struct {
	mu       sync.Mutex
	seen     map[string]struct{}
	inflight map[string]*tableEnsure
}

type tableEnsure struct {
	done chan struct{}
	err  error
}

func (c *tableCache) doOnce(key string, fn func() error) error {
	c.mu.Lock()
	if _, ok := c.seen[key]; ok {
		c.mu.Unlock()
		return nil
	}
	if c.inflight == nil {
		c.inflight = map[string]*tableEnsure{}
	}
	if entry, ok := c.inflight[key]; ok {
		c.mu.Unlock()
		<-entry.done
		return entry.err
	}
	entry := &tableEnsure{done: make(chan struct{})}
	c.inflight[key] = entry
	c.mu.Unlock()

	err := fn()

	c.mu.Lock()
	if err == nil {
		c.seen[key] = struct{}{}
	}
	entry.err = err
	delete(c.inflight, key)
	close(entry.done)
	c.mu.Unlock()
	return err
}

func NewWriter(exec Execer) *Writer {
	if exec == nil {
		panic("history execer must not be nil")
	}
	return &Writer{exec: exec, cache: tableCache{seen: map[string]struct{}{}}}
}

func NewWriterWithDatabase(exec Execer, database string) *Writer {
	writer := NewWriter(exec)
	writer.database = normalizeIdentifier(database)
	return writer
}

func (w *Writer) EnsureSchema(ctx context.Context, database string) error {
	for _, statement := range SchemaStatements(database) {
		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
			return err
		}
	}
	for _, statement := range SchemaMigrationStatements(database) {
		if _, err := w.exec.ExecContext(ctx, statement); err != nil && !isDuplicateTDengineColumnError(err) {
			return err
		}
	}
	return nil
}

func isDuplicateTDengineColumnError(err error) bool {
	if err == nil {
		return false
	}
	message := strings.ToLower(err.Error())
	return strings.Contains(message, "duplicate column") || strings.Contains(message, "duplicated column") || strings.Contains(message, "column already exists")
}

func (w *Writer) qualify(table string) string {
	table = normalizeIdentifier(table)
	if w.database == "" {
		return table
	}
	return w.database + "." + table
}

func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) error {
	result, err := w.AppendAllWithResult(ctx, env)
	if err != nil {
		return err
	}
	return result.LocationError
}

func (w *Writer) AppendAllWithResult(ctx context.Context, env envelope.FrameEnvelope) (AppendResult, error) {
	var result AppendResult
	if err := w.AppendRawFrame(ctx, env); err != nil {
		return result, err
	}
	result.RawRows = 1
	rows, err := w.appendLocationWithCount(ctx, env)
	if err != nil {
		result.LocationError = err
		return result, nil
	}
	result.LocationRows = rows
	return result, nil
}

func (w *Writer) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
	result, err := w.AppendAllBatchWithResult(ctx, envelopes)
	if err != nil {
		return err
	}
	return result.LocationError
}

func (w *Writer) AppendAllBatchWithResult(ctx context.Context, envelopes []envelope.FrameEnvelope) (AppendResult, error) {
	var result AppendResult
	if len(envelopes) == 0 {
		return result, nil
	}
	if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
		return result, err
	}
	result.RawRows = len(envelopes)
	rows, err := w.appendLocationBatchWithCount(ctx, envelopes)
	if err != nil {
		result.LocationError = err
		return result, nil
	}
	result.LocationRows = rows
	return result, nil
}

func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
	table := tableName("raw", env)
	if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
		return err
	}
	rawHex, rawHexChunks := chunkPayload(env, "raw_hex", env.RawHex)
	rawText, rawTextChunks := chunkPayload(env, "raw_text", env.RawText)
	parsedFields, parsedChunks := chunkPayload(env, "parsed_fields", parsedFieldsJSONString(env))
	_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
  (ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
   raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
VALUES (%s)`, w.qualify(table), joinLiterals(rawValues(env, rawHex, rawText, parsedFields))))
	if err != nil {
		return err
	}
	chunks := append(rawHexChunks, rawTextChunks...)
	chunks = append(chunks, parsedChunks...)
	if len(chunks) == 0 {
		return nil
	}
	chunkTable := tableName("chunk", env)
	if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
		return err
	}
	for _, chunk := range chunks {
		if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
  (ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
VALUES (%s)`, w.qualify(chunkTable), joinLiterals(chunkValues(env, chunk)))); err != nil {
			return err
		}
	}
	return nil
}

func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
	rowsByTable := map[string][]string{}
	chunkRowsByTable := map[string][]string{}
	for _, env := range envelopes {
		table := tableName("raw", env)
		if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
			return err
		}
		rawHex, rawHexChunks := chunkPayload(env, "raw_hex", env.RawHex)
		rawText, rawTextChunks := chunkPayload(env, "raw_text", env.RawText)
		parsedFields, parsedChunks := chunkPayload(env, "parsed_fields", parsedFieldsJSONString(env))
		rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(rawValues(env, rawHex, rawText, parsedFields))+")")
		chunks := append(rawHexChunks, rawTextChunks...)
		chunks = append(chunks, parsedChunks...)
		if len(chunks) == 0 {
			continue
		}
		chunkTable := tableName("chunk", env)
		if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
			return err
		}
		for _, chunk := range chunks {
			chunkRowsByTable[chunkTable] = append(chunkRowsByTable[chunkTable], "("+joinLiterals(chunkValues(env, chunk))+")")
		}
	}
	if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
   raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint`); err != nil {
		return err
	}
	if err := w.execMultiTableInsert(ctx, chunkRowsByTable, `ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text`); err != nil {
		return err
	}
	return nil
}

func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
	_, err := w.appendLocationWithCount(ctx, env)
	return err
}

func (w *Writer) appendLocationWithCount(ctx context.Context, env envelope.FrameEnvelope) (int, error) {
	longitude, latitude, status := locationCandidate(env)
	if status != LocationStatusOK {
		return 0, nil
	}
	table := locationTableName(env)
	if err := w.ensureLocationChild(ctx, table, env); err != nil {
		return 0, err
	}
	_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
  (ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
   soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES (%s)`, w.qualify(table), joinLiterals(locationValues(env, longitude, latitude))))
	if err != nil {
		return 0, err
	}
	return 1, nil
}

func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
	_, err := w.appendLocationBatchWithCount(ctx, envelopes)
	return err
}

func (w *Writer) appendLocationBatchWithCount(ctx context.Context, envelopes []envelope.FrameEnvelope) (int, error) {
	rowsByTable := map[string][]string{}
	rowCount := 0
	for _, env := range envelopes {
		longitude, latitude, status := locationCandidate(env)
		if status != LocationStatusOK {
			continue
		}
		table := locationTableName(env)
		if err := w.ensureLocationChild(ctx, table, env); err != nil {
			return rowCount, err
		}
		rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(locationValues(env, longitude, latitude))+")")
		rowCount++
	}
	if err := w.execMultiTableInsert(ctx, rowsByTable, `ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
   soc_percent, direction_deg, alarm_flag, status_flag, total_mileage_km`); err != nil {
		return rowCount, err
	}
	return rowCount, nil
}

func (w *Writer) execMultiTableInsert(ctx context.Context, rowsByTable map[string][]string, columns string) error {
	statements := buildMultiTableInsertStatements(rowsByTable, w.qualify, columns, tdengineInsertSoftLimit)
	for _, statement := range statements {
		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
			return err
		}
	}
	return nil
}

func buildMultiTableInsertStatements(rowsByTable map[string][]string, qualify func(string) string, columns string, softLimit int) []string {
	if len(rowsByTable) == 0 {
		return nil
	}
	keys := make([]string, 0, len(rowsByTable))
	for table, rows := range rowsByTable {
		if len(rows) > 0 {
			keys = append(keys, table)
		}
	}
	sort.Strings(keys)
	if len(keys) == 0 {
		return nil
	}
	if qualify == nil {
		qualify = func(table string) string { return table }
	}
	var statements []string
	current := "INSERT INTO "
	parts := 0
	for _, table := range keys {
		part := fmt.Sprintf(`%s
  (%s)
VALUES %s`, qualify(table), columns, strings.Join(rowsByTable[table], ","))
		if parts > 0 && softLimit > 0 && len(current)+1+len(part) > softLimit {
			statements = append(statements, current)
			current = "INSERT INTO "
			parts = 0
		}
		if parts > 0 {
			current += " "
		}
		current += part
		parts++
	}
	if parts > 0 {
		statements = append(statements, current)
	}
	return statements
}

func LocationStatus(env envelope.FrameEnvelope) string {
	_, _, status := locationCandidate(env)
	return status
}

func locationCandidate(env envelope.FrameEnvelope) (float64, float64, string) {
	if !envelope.IsRealtimeTelemetryFrame(env) {
		return 0, 0, LocationStatusSkippedNonRealtime
	}
	if strings.TrimSpace(env.VIN) == "" {
		return 0, 0, LocationStatusSkippedMissingVIN
	}
	location, ok := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
	if !ok {
		return 0, 0, LocationStatusSkippedMissingCoordinates
	}
	return location.Longitude, location.Latitude, LocationStatusOK
}

func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
	key := stable + "." + table
	return w.cache.doOnce(key, func() error {
		statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s', '%s', '%s', '%s')",
			w.qualify(table), w.qualify(stable),
			quote(string(env.Protocol)),
			quote(env.VehicleKey()),
			quote(env.VIN),
			quote(normalizePhoneTag(env.Phone)),
			quote(env.DeviceID))
		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
			return err
		}
		if phone := normalizePhoneTag(env.Phone); phone != "" {
			if _, err := w.exec.ExecContext(ctx, fmt.Sprintf("ALTER TABLE %s SET TAG phone = '%s'", w.qualify(table), quote(phone))); err != nil {
				return err
			}
		}
		return nil
	})
}

func (w *Writer) ensureLocationChild(ctx context.Context, table string, env envelope.FrameEnvelope) error {
	key := "vehicle_locations." + table
	return w.cache.doOnce(key, func() error {
		statement := fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s USING %s TAGS ('%s', '%s')",
			w.qualify(table),
			w.qualify("vehicle_locations"),
			quote(string(env.Protocol)),
			quote(strings.TrimSpace(env.VIN)))
		if _, err := w.exec.ExecContext(ctx, statement); err != nil {
			return err
		}
		return nil
	})
}

func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsedFields string) []any {
	received := millis(env.ReceivedAtMS)
	eventTime := millis(env.EventTimeMS)
	return []any{
		received,
		frameID(env),
		env.StableEventID(),
		messageIDInt(env.MessageID),
		eventTime,
		received,
		rawSizeBytes(env),
		rawHex,
		rawText,
		parsedFields,
		string(env.ParseStatus),
		env.ParseError,
		env.SourceEndpoint,
	}
}

func chunkValues(env envelope.FrameEnvelope, chunk payloadChunk) []any {
	received := millis(env.ReceivedAtMS)
	return []any{
		received.Add(time.Duration(chunk.Index) * time.Millisecond),
		env.StableEventID(),
		frameID(env),
		received,
		chunk.Kind,
		chunk.Index,
		chunk.Count,
		chunk.Text,
	}
}

func chunkPayload(env envelope.FrameEnvelope, kind string, value string) (string, []payloadChunk) {
	if len(value) <= rawFramePayloadInlineLimit {
		return value, nil
	}
	chunks := make([]payloadChunk, 0, (len(value)+rawFramePayloadChunkSize-1)/rawFramePayloadChunkSize)
	for index, start := 0, 0; start < len(value); index++ {
		end := safeChunkEnd(value, start, rawFramePayloadChunkSize)
		chunks = append(chunks, payloadChunk{
			Kind:  kind,
			Index: index,
			Text:  value[start:end],
		})
		start = end
	}
	count := len(chunks)
	for index := range chunks {
		chunks[index].Count = count
	}
	manifest := map[string]any{
		"chunked":      true,
		"payload_kind": kind,
		"event_id":     env.StableEventID(),
		"chunk_count":  count,
	}
	return jsonString(manifest), chunks
}

func safeChunkEnd(value string, start int, maxBytes int) int {
	end := start + maxBytes
	if end >= len(value) {
		return len(value)
	}
	for end > start && !utf8.RuneStart(value[end]) {
		end--
	}
	if end == start {
		return start + maxBytes
	}
	return end
}

func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
	received := millis(env.ReceivedAtMS)
	location, _ := telemetry.LocationProjectionForProtocol(env.Protocol, env.ParsedFields)
	totalMileageKM, hasTotalMileage := telemetry.TotalMileageKM(env.Protocol, env.ParsedFields)
	return []any{
		eventTimeOrReceived(env),
		env.StableEventID(),
		received,
		longitude,
		latitude,
		optionalFloat(location.AltitudeM),
		optionalFloat(location.SpeedKMH),
		optionalFloat(location.SOCPercent),
		optionalInt(location.DirectionDeg),
		optionalInt64(location.AlarmFlag),
		optionalInt64(location.StatusFlag),
		optionalPositiveFloat(totalMileageKM, hasTotalMileage),
	}
}

func frameID(env envelope.FrameEnvelope) string {
	return "go_" + env.StableEventID()
}

func tableName(prefix string, env envelope.FrameEnvelope) string {
	return prefix + "_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(env.VehicleKey())
}

func locationTableName(env envelope.FrameEnvelope) string {
	return "loc_" + strings.ToLower(string(env.Protocol)) + "_" + hash16(strings.TrimSpace(env.VIN))
}

func normalizeIdentifier(value string) string {
	value = strings.TrimSpace(value)
	value = strings.Trim(value, "`")
	return value
}

func normalizePhoneTag(phone string) string {
	trimmed := strings.TrimLeft(strings.TrimSpace(phone), "0")
	if trimmed == "" {
		return strings.TrimSpace(phone)
	}
	return trimmed
}

func hash16(value string) string {
	sum := sha1.Sum([]byte(value))
	return hex.EncodeToString(sum[:8])
}

func messageIDInt(value string) int64 {
	value = strings.TrimSpace(strings.ToLower(value))
	value = strings.TrimPrefix(value, "0x")
	parsed, err := strconv.ParseInt(value, 16, 32)
	if err == nil {
		return parsed
	}
	parsed, _ = strconv.ParseInt(value, 10, 32)
	return parsed
}

func rawSizeBytes(env envelope.FrameEnvelope) int {
	rawHex := strings.TrimSpace(env.RawHex)
	if len(rawHex)%2 != 0 {
		return len(rawHex) / 2
	}
	if rawHex != "" {
		return len(rawHex) / 2
	}
	return len([]byte(env.RawText))
}

func jsonString(value any) string {
	data, err := json.Marshal(value)
	if err != nil {
		return "{}"
	}
	return string(data)
}

func parsedFieldsJSONString(env envelope.FrameEnvelope) string {
	fields, _, ok := realtime.ParsedFieldsForEnvelope(env)
	if !ok || len(fields) == 0 {
		return ""
	}
	return jsonString(fields)
}

func optionalFloat(value *float64) any {
	if value == nil {
		return nil
	}
	return *value
}

func optionalInt(value *float64) any {
	if value == nil {
		return nil
	}
	return int64(*value)
}

func optionalInt64(value *int64) any {
	if value == nil {
		return nil
	}
	return *value
}

func optionalPositiveFloat(value float64, ok bool) any {
	if !ok || value <= 0 {
		return nil
	}
	return value
}

func eventTimeOrReceived(env envelope.FrameEnvelope) time.Time {
	eventMS, _ := envelope.NormalizedEventTimeMS(env)
	return millis(eventMS)
}

func millis(value int64) time.Time {
	if value <= 0 {
		return time.UnixMilli(time.Now().UnixMilli()).UTC()
	}
	return time.UnixMilli(value).UTC()
}

func quote(value string) string {
	return strings.NewReplacer(
		`\`, `\\`,
		`'`, `''`,
	).Replace(value)
}

func joinLiterals(values []any) string {
	out := make([]string, 0, len(values))
	for _, value := range values {
		out = append(out, literal(value))
	}
	return strings.Join(out, ", ")
}

func literal(value any) string {
	if value == nil {
		return "NULL"
	}
	switch typed := value.(type) {
	case time.Time:
		return strconv.FormatInt(typed.UnixMilli(), 10)
	case string:
		return "'" + quote(typed) + "'"
	case envelope.ParseStatus:
		return "'" + quote(string(typed)) + "'"
	case int:
		return strconv.FormatInt(int64(typed), 10)
	case int64:
		return strconv.FormatInt(typed, 10)
	case uint16:
		return strconv.FormatUint(uint64(typed), 10)
	case uint32:
		return strconv.FormatUint(uint64(typed), 10)
	case float64:
		return strconv.FormatFloat(typed, 'f', -1, 64)
	case float32:
		return strconv.FormatFloat(float64(typed), 'f', -1, 64)
	default:
		return "'" + quote(fmt.Sprint(typed)) + "'"
	}
}