feat(go): batch tdengine history writes

This commit is contained in:
lingniu
2026-07-03 18:37:38 +08:00
parent 3b0c5fbd5a
commit 09f3e68891
8 changed files with 447 additions and 12 deletions

View File

@@ -66,6 +66,16 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro
return w.AppendLocation(ctx, env)
}
func (w *Writer) AppendAllBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
if len(envelopes) == 0 {
return nil
}
if err := w.AppendRawFrameBatch(ctx, envelopes); err != nil {
return err
}
return w.AppendLocationBatch(ctx, envelopes)
}
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
table := tableName("raw", env)
if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
@@ -100,6 +110,58 @@ VALUES (%s)`, chunkTable, joinLiterals(chunkValues(env, chunk)))); err != nil {
return nil
}
func (w *Writer) AppendRawFrameBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
rowsByTable := map[string][]string{}
chunkRowsByTable := map[string][]string{}
chunkEnvByTable := map[string]envelope.FrameEnvelope{}
for _, env := range envelopes {
table := tableName("raw", env)
if err := w.ensureRawChild(ctx, table, "raw_frames", env); err != nil {
return err
}
rawHex, rawHexChunks := chunkPayload(env, "raw_hex", env.RawHex)
rawText, rawTextChunks := chunkPayload(env, "raw_text", env.RawText)
parsedFields, parsedChunks := chunkPayload(env, "parsed_fields", parsedFieldsJSONString(env))
rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(rawValues(env, rawHex, rawText, parsedFields))+")")
chunks := append(rawHexChunks, rawTextChunks...)
chunks = append(chunks, parsedChunks...)
if len(chunks) == 0 {
continue
}
chunkTable := tableName("chunk", env)
if err := w.ensureRawChild(ctx, chunkTable, "raw_frame_payload_chunks", env); err != nil {
return err
}
chunkEnvByTable[chunkTable] = env
for _, chunk := range chunks {
chunkRowsByTable[chunkTable] = append(chunkRowsByTable[chunkTable], "("+joinLiterals(chunkValues(env, chunk))+")")
}
}
for table, rows := range rowsByTable {
if len(rows) == 0 {
continue
}
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes,
raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint)
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
return err
}
}
for table, rows := range chunkRowsByTable {
if len(rows) == 0 {
continue
}
_ = chunkEnvByTable[table]
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, payload_kind, chunk_index, chunk_count, chunk_text)
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
return err
}
}
return nil
}
func (w *Writer) AppendLocation(ctx context.Context, env envelope.FrameEnvelope) error {
if strings.TrimSpace(env.VIN) == "" {
return nil
@@ -120,6 +182,37 @@ VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
return err
}
func (w *Writer) AppendLocationBatch(ctx context.Context, envelopes []envelope.FrameEnvelope) error {
rowsByTable := map[string][]string{}
for _, env := range envelopes {
if strings.TrimSpace(env.VIN) == "" {
continue
}
longitude, okLon := floatField(env, envelope.FieldLongitude)
latitude, okLat := floatField(env, envelope.FieldLatitude)
if !okLon || !okLat {
continue
}
table := locationTableName(env)
if err := w.ensureLocationChild(ctx, table, env); err != nil {
return err
}
rowsByTable[table] = append(rowsByTable[table], "("+joinLiterals(locationValues(env, longitude, latitude))+")")
}
for table, rows := range rowsByTable {
if len(rows) == 0 {
continue
}
if _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, received_at, longitude, latitude, altitude_m, speed_kmh,
direction_deg, alarm_flag, status_flag, total_mileage_km)
VALUES %s`, table, strings.Join(rows, ","))); err != nil {
return err
}
}
return nil
}
func (w *Writer) ensureRawChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
key := stable + "." + table
w.cache.mu.Lock()

View File

@@ -226,6 +226,53 @@ func TestWriterSkipsLocationWhenVINIsMissing(t *testing.T) {
}
}
func TestWriterAppendsBatchRowsByChildTable(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
first := sampleEnvelope()
second := sampleEnvelope()
second.Sequence = 2
second.EventTimeMS += 1000
second.ReceivedAtMS += 1000
if err := writer.AppendAllBatch(context.Background(), []envelope.FrameEnvelope{first, second}); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if got := countSQL(exec.calls, "USING raw_frames"); got != 1 {
t.Fatalf("raw child create count = %d", got)
}
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
t.Fatalf("location child create count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw batch insert count = %d", got)
}
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location batch insert count = %d", got)
}
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if got := strings.Count(rawInsert, "),(") + 1; got != 2 {
t.Fatalf("raw batch row count = %d, sql=%s", got, rawInsert)
}
locationInsert := findSQL(exec.calls, "INSERT INTO loc_")
if got := strings.Count(locationInsert, "),(") + 1; got != 2 {
t.Fatalf("location batch row count = %d, sql=%s", got, locationInsert)
}
}
func TestWriterAppendAllBatchSkipsEmptyBatch(t *testing.T) {
exec := &recordingExec{}
writer := NewWriter(exec)
if err := writer.AppendAllBatch(context.Background(), nil); err != nil {
t.Fatalf("AppendAllBatch() error = %v", err)
}
if len(exec.calls) != 0 {
t.Fatalf("exec calls = %#v, want none", exec.calls)
}
}
func TestRawSizeBytesUsesRawTextWhenHexIsEmpty(t *testing.T) {
env := envelope.FrameEnvelope{RawText: `{"code":"0F80","data":{"speed":12}}`}
if got, want := rawSizeBytes(env), len([]byte(env.RawText)); got != want {