feat(go): persist full raw payload chunks

go/vehicle-gateway/internal/history/writer.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unicode/utf8"
 
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
 )
@@ -24,6 +25,18 @@ type Writer struct {
 	cache tableCache
 }
 
+const (
+	rawFramePayloadInlineLimit = 12_000
+	rawFramePayloadChunkSize   = 16_000
+)
+
+type payloadChunk struct {
+	Kind  string
+	Index int
+	Count int
+	Text  string
+}
+
 type tableCache struct {
 	mu   sync.Mutex
 	seen map[string]struct{}
@@ -60,11 +73,35 @@ func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope)
 	if err := w.ensureChild(ctx, table, "raw_frames", env); err != nil {
 		return err
 	}
+	rawHex, rawHexChunks := chunkPayload(env, "raw_hex", env.RawHex)
+	rawText, rawTextChunks := chunkPayload(env, "raw_text", env.RawText)
+	parsedJSON, parsedChunks := chunkPayload(env, "parsed_json", jsonString(env.Parsed))
+	fieldsJSON, fieldsChunks := chunkPayload(env, "fields_json", jsonString(env.Fields))
 	_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
   (ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
    raw_hex, raw_text, parsed_json, fields_json, parse_status, parse_error, source_endpoint)
-VALUES (%s)`, table, joinLiterals(rawValues(env))))
-	return err
+VALUES (%s)`, table, joinLiterals(rawValues(env, rawHex, rawText, parsedJSON, fieldsJSON))))
+	if err != nil {
+		return err
+	}
+	chunks := append(rawHexChunks, rawTextChunks...)
+	chunks = append(chunks, parsedChunks...)
+	chunks = append(chunks, fieldsChunks...)
+	if len(chunks) == 0 {
+		return nil
+	}
+	chunkTable := tableName("chunk", env)
+	if err := w.ensureChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
+		return err
+	}
+	for _, chunk := range chunks {
+		if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
+  (ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
+VALUES (%s)`, chunkTable, joinLiterals(chunkValues(env, chunk)))); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -123,7 +160,7 @@ func (w *Writer) ensureChild(ctx context.Context, table string, stable string, e
 	return nil
 }
 
-func rawValues(env envelope.FrameEnvelope) []any {
+func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsedJSON string, fieldsJSON string) []any {
 	received := millis(env.ReceivedAtMS)
 	eventTime := millis(env.EventTimeMS)
 	return []any{
@@ -134,16 +171,71 @@ func rawValues(env envelope.FrameEnvelope) []any {
 		eventTime,
 		received,
 		rawSizeBytes(env),
-		env.RawHex,
-		env.RawText,
-		jsonString(env.Parsed),
-		jsonString(env.Fields),
+		rawHex,
+		rawText,
+		parsedJSON,
+		fieldsJSON,
 		string(env.ParseStatus),
 		env.ParseError,
 		env.SourceEndpoint,
 	}
 }
 
+func chunkValues(env envelope.FrameEnvelope, chunk payloadChunk) []any {
+	received := millis(env.ReceivedAtMS)
+	return []any{
+		received,
+		env.StableEventID(),
+		frameID(env),
+		received,
+		chunk.Kind,
+		chunk.Index,
+		chunk.Count,
+		chunk.Text,
+	}
+}
+
+func chunkPayload(env envelope.FrameEnvelope, kind string, value string) (string, []payloadChunk) {
+	if len(value) <= rawFramePayloadInlineLimit {
+		return value, nil
+	}
+	chunks := make([]payloadChunk, 0, (len(value)+rawFramePayloadChunkSize-1)/rawFramePayloadChunkSize)
+	for index, start := 0, 0; start < len(value); index++ {
+		end := safeChunkEnd(value, start, rawFramePayloadChunkSize)
+		chunks = append(chunks, payloadChunk{
+			Kind:  kind,
+			Index: index,
+			Text:  value[start:end],
+		})
+		start = end
+	}
+	count := len(chunks)
+	for index := range chunks {
+		chunks[index].Count = count
+	}
+	manifest := map[string]any{
+		"chunked":      true,
+		"payload_kind": kind,
+		"event_id":     env.StableEventID(),
+		"chunk_count":  count,
+	}
+	return jsonString(manifest), chunks
+}
+
+func safeChunkEnd(value string, start int, maxBytes int) int {
+	end := start + maxBytes
+	if end >= len(value) {
+		return len(value)
+	}
+	for end > start && !utf8.RuneStart(value[end]) {
+		end--
+	}
+	if end == start {
+		return start + maxBytes
+	}
+	return end
+}
+
 func locationValues(env envelope.FrameEnvelope, longitude float64, latitude float64) []any {
 	received := millis(env.ReceivedAtMS)
 	return []any{