feat(go): persist full raw payload chunks

This commit is contained in:
lingniu
2026-07-02 12:11:24 +08:00
parent 3a67f6e05f
commit 7a9a6f441a
7 changed files with 334 additions and 12 deletions

View File

@@ -206,7 +206,13 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
out = append(out, row)
}
return out, rows.Err()
if err := rows.Err(); err != nil {
return nil, err
}
if err := r.hydratePayloadChunks(ctx, out); err != nil {
return nil, err
}
return out, nil
}
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
@@ -347,6 +353,13 @@ func (r *RawFrameRepository) tableName() string {
return r.database + ".raw_frames"
}
func (r *RawFrameRepository) chunkTableName() string {
if r.database == "" {
return "raw_frame_payload_chunks"
}
return r.database + ".raw_frame_payload_chunks"
}
func (r *LocationRepository) tableName() string {
if r.database == "" {
return "vehicle_locations"
@@ -423,6 +436,129 @@ func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
return sqlText, nil
}
type payloadChunkManifest struct {
Chunked bool `json:"chunked"`
PayloadKind string `json:"payload_kind"`
EventID string `json:"event_id"`
ChunkCount int `json:"chunk_count"`
}
func (r *RawFrameRepository) hydratePayloadChunks(ctx context.Context, rows []RawFrameRow) error {
type target struct {
rowIndex int
kind string
eventID string
count int
}
targets := make([]target, 0)
eventSeen := map[string]struct{}{}
for index := range rows {
for _, candidate := range []struct {
kind string
value string
}{
{kind: "raw_hex", value: rows[index].RawHex},
{kind: "raw_text", value: rows[index].RawText},
{kind: "parsed_json", value: rows[index].ParsedJSON},
{kind: "fields_json", value: rows[index].FieldsJSON},
} {
manifest, ok := parsePayloadChunkManifest(candidate.value)
if !ok || manifest.PayloadKind != candidate.kind || manifest.ChunkCount <= 0 {
continue
}
eventID := strings.TrimSpace(manifest.EventID)
if eventID == "" {
eventID = rows[index].EventID
}
targets = append(targets, target{
rowIndex: index,
kind: candidate.kind,
eventID: eventID,
count: manifest.ChunkCount,
})
eventSeen[eventID] = struct{}{}
}
}
if len(targets) == 0 {
return nil
}
eventIDs := make([]string, 0, len(eventSeen))
for eventID := range eventSeen {
eventIDs = append(eventIDs, eventID)
}
sqlText := `SELECT event_id, payload_kind, chunk_index, chunk_text FROM ` + r.chunkTableName() +
" WHERE event_id IN (" + quotedList(eventIDs) + ") ORDER BY event_id, payload_kind, chunk_index"
chunkRows, err := r.db.QueryContext(ctx, sqlText)
if err != nil {
return err
}
defer chunkRows.Close()
chunks := map[string]map[string][]string{}
for chunkRows.Next() {
var eventID string
var kind string
var index int
var text string
if err := chunkRows.Scan(&eventID, &kind, &index, &text); err != nil {
return err
}
if chunks[eventID] == nil {
chunks[eventID] = map[string][]string{}
}
current := chunks[eventID][kind]
if index >= len(current) {
next := make([]string, index+1)
copy(next, current)
current = next
}
current[index] = text
chunks[eventID][kind] = current
}
if err := chunkRows.Err(); err != nil {
return err
}
for _, target := range targets {
parts := chunks[target.eventID][target.kind]
if len(parts) != target.count {
continue
}
hydrated := strings.Join(parts, "")
switch target.kind {
case "raw_hex":
rows[target.rowIndex].RawHex = hydrated
case "raw_text":
rows[target.rowIndex].RawText = hydrated
case "parsed_json":
rows[target.rowIndex].ParsedJSON = hydrated
case "fields_json":
rows[target.rowIndex].FieldsJSON = hydrated
}
}
return nil
}
func parsePayloadChunkManifest(value string) (payloadChunkManifest, bool) {
var manifest payloadChunkManifest
if !strings.Contains(value, `"chunked"`) {
return manifest, false
}
if err := json.Unmarshal([]byte(value), &manifest); err != nil {
return manifest, false
}
return manifest, manifest.Chunked
}
func quotedList(values []string) string {
out := make([]string, 0, len(values))
for _, value := range values {
out = append(out, "'"+quote(value)+"'")
}
return strings.Join(out, ", ")
}
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
where := locationWhere(query)
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table