feat(go): persist full raw payload chunks
This commit is contained in:
@@ -206,7 +206,13 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
|
||||
row.MessageIDHex = fmt.Sprintf("0x%04X", row.MessageID)
|
||||
out = append(out, row)
|
||||
}
|
||||
return out, rows.Err()
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := r.hydratePayloadChunks(ctx, out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
|
||||
@@ -347,6 +353,13 @@ func (r *RawFrameRepository) tableName() string {
|
||||
return r.database + ".raw_frames"
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) chunkTableName() string {
|
||||
if r.database == "" {
|
||||
return "raw_frame_payload_chunks"
|
||||
}
|
||||
return r.database + ".raw_frame_payload_chunks"
|
||||
}
|
||||
|
||||
func (r *LocationRepository) tableName() string {
|
||||
if r.database == "" {
|
||||
return "vehicle_locations"
|
||||
@@ -423,6 +436,129 @@ func buildRawFrameCountSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
return sqlText, nil
|
||||
}
|
||||
|
||||
type payloadChunkManifest struct {
|
||||
Chunked bool `json:"chunked"`
|
||||
PayloadKind string `json:"payload_kind"`
|
||||
EventID string `json:"event_id"`
|
||||
ChunkCount int `json:"chunk_count"`
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) hydratePayloadChunks(ctx context.Context, rows []RawFrameRow) error {
|
||||
type target struct {
|
||||
rowIndex int
|
||||
kind string
|
||||
eventID string
|
||||
count int
|
||||
}
|
||||
targets := make([]target, 0)
|
||||
eventSeen := map[string]struct{}{}
|
||||
for index := range rows {
|
||||
for _, candidate := range []struct {
|
||||
kind string
|
||||
value string
|
||||
}{
|
||||
{kind: "raw_hex", value: rows[index].RawHex},
|
||||
{kind: "raw_text", value: rows[index].RawText},
|
||||
{kind: "parsed_json", value: rows[index].ParsedJSON},
|
||||
{kind: "fields_json", value: rows[index].FieldsJSON},
|
||||
} {
|
||||
manifest, ok := parsePayloadChunkManifest(candidate.value)
|
||||
if !ok || manifest.PayloadKind != candidate.kind || manifest.ChunkCount <= 0 {
|
||||
continue
|
||||
}
|
||||
eventID := strings.TrimSpace(manifest.EventID)
|
||||
if eventID == "" {
|
||||
eventID = rows[index].EventID
|
||||
}
|
||||
targets = append(targets, target{
|
||||
rowIndex: index,
|
||||
kind: candidate.kind,
|
||||
eventID: eventID,
|
||||
count: manifest.ChunkCount,
|
||||
})
|
||||
eventSeen[eventID] = struct{}{}
|
||||
}
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
eventIDs := make([]string, 0, len(eventSeen))
|
||||
for eventID := range eventSeen {
|
||||
eventIDs = append(eventIDs, eventID)
|
||||
}
|
||||
sqlText := `SELECT event_id, payload_kind, chunk_index, chunk_text FROM ` + r.chunkTableName() +
|
||||
" WHERE event_id IN (" + quotedList(eventIDs) + ") ORDER BY event_id, payload_kind, chunk_index"
|
||||
chunkRows, err := r.db.QueryContext(ctx, sqlText)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer chunkRows.Close()
|
||||
|
||||
chunks := map[string]map[string][]string{}
|
||||
for chunkRows.Next() {
|
||||
var eventID string
|
||||
var kind string
|
||||
var index int
|
||||
var text string
|
||||
if err := chunkRows.Scan(&eventID, &kind, &index, &text); err != nil {
|
||||
return err
|
||||
}
|
||||
if chunks[eventID] == nil {
|
||||
chunks[eventID] = map[string][]string{}
|
||||
}
|
||||
current := chunks[eventID][kind]
|
||||
if index >= len(current) {
|
||||
next := make([]string, index+1)
|
||||
copy(next, current)
|
||||
current = next
|
||||
}
|
||||
current[index] = text
|
||||
chunks[eventID][kind] = current
|
||||
}
|
||||
if err := chunkRows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, target := range targets {
|
||||
parts := chunks[target.eventID][target.kind]
|
||||
if len(parts) != target.count {
|
||||
continue
|
||||
}
|
||||
hydrated := strings.Join(parts, "")
|
||||
switch target.kind {
|
||||
case "raw_hex":
|
||||
rows[target.rowIndex].RawHex = hydrated
|
||||
case "raw_text":
|
||||
rows[target.rowIndex].RawText = hydrated
|
||||
case "parsed_json":
|
||||
rows[target.rowIndex].ParsedJSON = hydrated
|
||||
case "fields_json":
|
||||
rows[target.rowIndex].FieldsJSON = hydrated
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func parsePayloadChunkManifest(value string) (payloadChunkManifest, bool) {
|
||||
var manifest payloadChunkManifest
|
||||
if !strings.Contains(value, `"chunked"`) {
|
||||
return manifest, false
|
||||
}
|
||||
if err := json.Unmarshal([]byte(value), &manifest); err != nil {
|
||||
return manifest, false
|
||||
}
|
||||
return manifest, manifest.Chunked
|
||||
}
|
||||
|
||||
func quotedList(values []string) string {
|
||||
out := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
out = append(out, "'"+quote(value)+"'")
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
func buildLocationSQL(table string, query LocationQuery) (string, []any) {
|
||||
where := locationWhere(query)
|
||||
sqlText := `SELECT ts, event_id, frame_id, received_at, longitude, latitude, altitude_m, speed_kmh, direction_deg, alarm_flag, status_flag, total_mileage_km, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
|
||||
Reference in New Issue
Block a user