shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(go): optimize tdengine raw frame queries

Browse Source
This commit is contained in:
lingniu
2026-07-03 14:25:48 +08:00
parent 535acdb2a6
commit 6318c10c41
4 changed files with 123 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
go/vehicle-gateway/internal/history/writer.go
View File

@@ -14,6 +14,7 @@ import (
"unicode/utf8"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
)
type Execer interface {
@@ -72,11 +73,11 @@ func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope)
}
rawHex, rawHexChunks := chunkPayload(env, "raw_hex", env.RawHex)
rawText, rawTextChunks := chunkPayload(env, "raw_text", env.RawText)
parsedJSON, parsedChunks := chunkPayload(env, "parsed_json", parsedJSONString(env.Parsed))
parsedFields, parsedChunks := chunkPayload(env, "parsed_fields", parsedFieldsJSONString(env))
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
VALUES (%s)`, table, joinLiterals(rawValues(env, rawHex, rawText, parsedJSON))))
VALUES (%s)`, table, joinLiterals(rawValues(env, rawHex, rawText, parsedFields))))
if err != nil {
return err
}
@@ -169,7 +170,7 @@ func (w *Writer) ensureLocationChild(ctx context.Context, table string, env enve
return nil
}
func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsedJSON string) []any {
func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsedFields string) []any {
received := millis(env.ReceivedAtMS)
eventTime := millis(env.EventTimeMS)
return []any{
@@ -182,7 +183,7 @@ func rawValues(env envelope.FrameEnvelope, rawHex string, rawText string, parsed
rawSizeBytes(env),
rawHex,
rawText,
parsedJSON,
parsedFields,
string(env.ParseStatus),
env.ParseError,
env.SourceEndpoint,
@@ -316,11 +317,12 @@ func jsonString(value any) string {
return string(data)
}
func parsedJSONString(value map[string]any) string {
if len(value) == 0 {
func parsedFieldsJSONString(env envelope.FrameEnvelope) string {
fieldsEnv, ok := realtime.BuildFieldsEnvelope(env)
if !ok || len(fieldsEnv.Fields) == 0 {
return ""
}
return jsonString(value)
return jsonString(fieldsEnv.Fields)
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {