perf(go): optimize tdengine raw frame queries
This commit is contained in:
lingniu
@@ -17,18 +17,20 @@ type Queryer interface {
|
||||
}
|
||||
|
||||
type RawFrameQuery struct {
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
MessageID string
|
||||
OrderBy string
|
||||
IncludeTotal bool
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
Protocol string
|
||||
VehicleKey string
|
||||
VIN string
|
||||
Phone string
|
||||
DeviceID string
|
||||
MessageID string
|
||||
OrderBy string
|
||||
IncludeFields bool
|
||||
IncludePayload bool
|
||||
IncludeTotal bool
|
||||
DateFrom string
|
||||
DateTo string
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
type RawFrameRow struct {
|
||||
@@ -42,7 +44,7 @@ type RawFrameRow struct {
|
||||
RawSizeBytes int64 `json:"raw_size_bytes"`
|
||||
RawHex string `json:"raw_hex,omitempty"`
|
||||
RawText string `json:"raw_text,omitempty"`
|
||||
ParsedJSON string `json:"parsed_json,omitempty"`
|
||||
ParsedFields string `json:"parsed_fields,omitempty"`
|
||||
ParseStatus string `json:"parse_status"`
|
||||
ParseError string `json:"parse_error,omitempty"`
|
||||
SourceEndpoint string `json:"source_endpoint"`
|
||||
@@ -113,7 +115,7 @@ func NewLocationRepository(db Queryer, database string) *LocationRepository {
|
||||
|
||||
func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]RawFrameRow, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameSQL(r.tableName(), query)
|
||||
sqlText, args := buildRawFrameSQL(r.tableName(query), query)
|
||||
rows, err := r.db.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -136,7 +138,7 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
|
||||
&row.RawSizeBytes,
|
||||
&row.RawHex,
|
||||
&row.RawText,
|
||||
&row.ParsedJSON,
|
||||
&row.ParsedFields,
|
||||
&row.ParseStatus,
|
||||
&row.ParseError,
|
||||
&row.SourceEndpoint,
|
||||
@@ -165,7 +167,7 @@ func (r *RawFrameRepository) Query(ctx context.Context, query RawFrameQuery) ([]
|
||||
|
||||
func (r *RawFrameRepository) Count(ctx context.Context, query RawFrameQuery) (int64, error) {
|
||||
query = normalizeRawFrameQuery(query)
|
||||
sqlText, args := buildRawFrameCountSQL(r.tableName(), query)
|
||||
sqlText, args := buildRawFrameCountSQL(r.tableName(query), query)
|
||||
return countRows(ctx, r.db, sqlText, args...)
|
||||
}
|
||||
|
||||
@@ -240,13 +242,37 @@ func countRows(ctx context.Context, db Queryer, sqlText string, args ...any) (in
|
||||
return total, rows.Err()
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) tableName() string {
|
||||
func (r *RawFrameRepository) tableName(query ...RawFrameQuery) string {
|
||||
if len(query) > 0 {
|
||||
if child := r.rawChildTableName(query[0]); child != "" {
|
||||
return child
|
||||
}
|
||||
}
|
||||
if r.database == "" {
|
||||
return "raw_frames"
|
||||
}
|
||||
return r.database + ".raw_frames"
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) rawChildTableName(query RawFrameQuery) string {
|
||||
protocol := strings.ToUpper(strings.TrimSpace(query.Protocol))
|
||||
if protocol == "" {
|
||||
return ""
|
||||
}
|
||||
vehicleKey := strings.TrimSpace(query.VehicleKey)
|
||||
if vehicleKey == "" {
|
||||
vehicleKey = strings.TrimSpace(query.VIN)
|
||||
}
|
||||
if vehicleKey == "" {
|
||||
return ""
|
||||
}
|
||||
table := "raw_" + strings.ToLower(protocol) + "_" + hash16(vehicleKey)
|
||||
if r.database != "" {
|
||||
return r.database + "." + table
|
||||
}
|
||||
return table
|
||||
}
|
||||
|
||||
func (r *RawFrameRepository) chunkTableName() string {
|
||||
if r.database == "" {
|
||||
return "raw_frame_payload_chunks"
|
||||
@@ -290,7 +316,17 @@ func normalizeLocationQuery(query LocationQuery) LocationQuery {
|
||||
|
||||
func buildRawFrameSQL(table string, query RawFrameQuery) (string, []any) {
|
||||
where := rawFrameWhere(query)
|
||||
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
rawHexSelect := "'' AS raw_hex"
|
||||
rawTextSelect := "'' AS raw_text"
|
||||
parsedFieldsSelect := "'' AS parsed_fields"
|
||||
if query.IncludePayload {
|
||||
rawHexSelect = "raw_hex"
|
||||
rawTextSelect = "raw_text"
|
||||
}
|
||||
if query.IncludeFields {
|
||||
parsedFieldsSelect = "parsed_json AS parsed_fields"
|
||||
}
|
||||
sqlText := `SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, ` + rawHexSelect + `, ` + rawTextSelect + `, ` + parsedFieldsSelect + `, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM ` + table
|
||||
if len(where) > 0 {
|
||||
sqlText += " WHERE " + strings.Join(where, " AND ")
|
||||
}
|
||||
@@ -329,10 +365,10 @@ func (r *RawFrameRepository) hydratePayloadChunks(ctx context.Context, rows []Ra
|
||||
}{
|
||||
{kind: "raw_hex", value: rows[index].RawHex},
|
||||
{kind: "raw_text", value: rows[index].RawText},
|
||||
{kind: "parsed_json", value: rows[index].ParsedJSON},
|
||||
{kind: "parsed_fields", value: rows[index].ParsedFields},
|
||||
} {
|
||||
manifest, ok := parsePayloadChunkManifest(candidate.value)
|
||||
if !ok || manifest.PayloadKind != candidate.kind || manifest.ChunkCount <= 0 {
|
||||
if !ok || !payloadKindMatches(candidate.kind, manifest.PayloadKind) || manifest.ChunkCount <= 0 {
|
||||
continue
|
||||
}
|
||||
eventID := strings.TrimSpace(manifest.EventID)
|
||||
@@ -400,13 +436,20 @@ func (r *RawFrameRepository) hydratePayloadChunks(ctx context.Context, rows []Ra
|
||||
rows[target.rowIndex].RawHex = hydrated
|
||||
case "raw_text":
|
||||
rows[target.rowIndex].RawText = hydrated
|
||||
case "parsed_json":
|
||||
rows[target.rowIndex].ParsedJSON = hydrated
|
||||
case "parsed_fields":
|
||||
rows[target.rowIndex].ParsedFields = hydrated
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func payloadKindMatches(candidate string, manifest string) bool {
|
||||
if candidate == manifest {
|
||||
return true
|
||||
}
|
||||
return candidate == "parsed_fields" && manifest == "parsed_json"
|
||||
}
|
||||
|
||||
func parsePayloadChunkManifest(value string) (payloadChunkManifest, bool) {
|
||||
var manifest payloadChunkManifest
|
||||
if !strings.Contains(value, `"chunked"`) {
|
||||
@@ -626,18 +669,20 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
|
||||
return RawFrameQuery{}, err
|
||||
}
|
||||
query := RawFrameQuery{
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
MessageID: values.Get("messageId"),
|
||||
OrderBy: values.Get("orderBy"),
|
||||
IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
Protocol: values.Get("protocol"),
|
||||
VehicleKey: values.Get("vehicleKey"),
|
||||
VIN: values.Get("vin"),
|
||||
Phone: values.Get("phone"),
|
||||
DeviceID: values.Get("deviceId"),
|
||||
MessageID: values.Get("messageId"),
|
||||
OrderBy: values.Get("orderBy"),
|
||||
IncludeFields: strings.EqualFold(strings.TrimSpace(values.Get("includeFields")), "true"),
|
||||
IncludePayload: strings.EqualFold(strings.TrimSpace(values.Get("includePayload")), "true"),
|
||||
IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
|
||||
DateFrom: values.Get("dateFrom"),
|
||||
DateTo: values.Get("dateTo"),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||
|
||||
Reference in New Issue
Block a user