feat(go): filter raw parsed fields

This commit is contained in:
lingniu
2026-07-03 14:49:53 +08:00
parent a0a4997e79
commit c0d3b41550
2 changed files with 93 additions and 0 deletions

View File

@@ -28,6 +28,7 @@ type RawFrameQuery struct {
IncludeFields bool
IncludePayload bool
IncludeTotal bool
ParsedFields []string
DateFrom string
DateTo string
Limit int
@@ -165,6 +166,7 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
if err := r.hydratePayloadChunks(ctx, out); err != nil {
return nil, err
}
filterParsedFields(out, query.ParsedFields)
return out, nil
}
@@ -297,6 +299,10 @@ func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
query.Phone = strings.TrimSpace(query.Phone)
query.DeviceID = strings.TrimSpace(query.DeviceID)
query.MessageID = strings.TrimSpace(query.MessageID)
query.ParsedFields = normalizeParsedFieldNames(query.ParsedFields)
if len(query.ParsedFields) > 0 {
query.IncludeFields = true
}
query.OrderBy = normalizeRawFrameOrderBy(query.OrderBy)
query.DateFrom = normalizeDateTimeLiteral(query.DateFrom)
query.DateTo = normalizeDateTimeLiteral(query.DateTo)
@@ -457,6 +463,52 @@ func rawJSONMessage(value string) json.RawMessage {
return json.RawMessage(jsonString(value))
}
func filterParsedFields(rows []RawFrameRow, fieldNames []string) {
fieldNames = normalizeParsedFieldNames(fieldNames)
if len(fieldNames) == 0 {
return
}
for index := range rows {
if len(rows[index].ParsedFields) == 0 {
continue
}
var fields map[string]any
if err := json.Unmarshal(rows[index].ParsedFields, &fields); err != nil {
continue
}
filtered := make(map[string]any, len(fieldNames))
for _, name := range fieldNames {
if value, ok := fields[name]; ok {
filtered[name] = value
}
}
if len(filtered) == 0 {
rows[index].ParsedFields = json.RawMessage(`{}`)
continue
}
rows[index].ParsedFields = rawJSONMessage(jsonString(filtered))
}
}
func normalizeParsedFieldNames(values []string) []string {
out := make([]string, 0, len(values))
seen := map[string]struct{}{}
for _, value := range values {
for _, part := range strings.Split(value, ",") {
name := strings.TrimSpace(part)
if name == "" {
continue
}
if _, ok := seen[name]; ok {
continue
}
seen[name] = struct{}{}
out = append(out, name)
}
}
return out
}
func payloadKindMatches(candidate string, manifest string) bool {
if candidate == manifest {
return true
@@ -704,6 +756,7 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
IncludeFields: strings.EqualFold(strings.TrimSpace(values.Get("includeFields")), "true"),
IncludePayload: strings.EqualFold(strings.TrimSpace(values.Get("includePayload")), "true"),
IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
ParsedFields: append(values["fields"], values["parsedFields"]...),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,