feat(go): batch tdengine history writes

go/vehicle-gateway/cmd/history-writer/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"strings"
 	"testing"
 
@@ -74,6 +75,89 @@ func TestProcessHistoryMessageRecordsMetrics(t *testing.T) {
 	}
 }
 
+func TestProcessHistoryBatchAppendsAllBeforeCommit(t *testing.T) {
+	first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
+	second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
+	firstPayload, err := json.Marshal(first)
+	if err != nil {
+		t.Fatal(err)
+	}
+	secondPayload, err := json.Marshal(second)
+	if err != nil {
+		t.Fatal(err)
+	}
+	appender := &contextCheckingHistoryAppender{}
+	committer := &contextCheckingHistoryCommitter{}
+	registry := metrics.NewRegistry()
+
+	processHistoryBatch(
+		context.Background(),
+		discardHistoryLogger{},
+		registry,
+		appender,
+		committer,
+		[]kafka.Message{
+			{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 13, Value: firstPayload},
+			{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 20, HighWaterMark: 21, Value: secondPayload},
+		},
+	)
+
+	if appender.batchCount != 1 {
+		t.Fatalf("batch appends = %d, want 1", appender.batchCount)
+	}
+	if got := len(appender.batch); got != 2 {
+		t.Fatalf("batch size = %d, want 2", got)
+	}
+	if committer.count != 1 {
+		t.Fatalf("commit calls = %d, want 1", committer.count)
+	}
+	if committer.messageCount != 2 {
+		t.Fatalf("committed messages = %d, want 2", committer.messageCount)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_history_batch_rows_total{status="ok"} 2`,
+		`vehicle_history_batch_flush_total{status="ok"} 1`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("batch metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
+func TestProcessHistoryBatchDoesNotCommitWhenAppendFails(t *testing.T) {
+	env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
+	payload, err := json.Marshal(env)
+	if err != nil {
+		t.Fatal(err)
+	}
+	appender := &contextCheckingHistoryAppender{err: errTestHistoryAppend}
+	committer := &contextCheckingHistoryCommitter{}
+	registry := metrics.NewRegistry()
+
+	processHistoryBatch(
+		context.Background(),
+		discardHistoryLogger{},
+		registry,
+		appender,
+		committer,
+		[]kafka.Message{{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}},
+	)
+
+	if committer.count != 0 {
+		t.Fatalf("commit calls = %d, want 0", committer.count)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_history_batch_rows_total{status="error"} 1`,
+		`vehicle_history_batch_flush_total{status="error"} 1`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("batch error metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
 func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
 	cfg := loadConfig()
 
@@ -82,27 +166,65 @@ func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
 	if got != want {
 		t.Fatalf("KafkaTopics = %q, want %q", got, want)
 	}
+	if cfg.BatchSize != 200 {
+		t.Fatalf("BatchSize = %d, want 200", cfg.BatchSize)
+	}
+	if cfg.BatchWait != 100 {
+		t.Fatalf("BatchWait = %d, want 100", cfg.BatchWait)
+	}
+}
+
+func TestLoadConfigReadsBatchSettings(t *testing.T) {
+	t.Setenv("HISTORY_BATCH_SIZE", "500")
+	t.Setenv("HISTORY_BATCH_WAIT_MS", "250")
+
+	cfg := loadConfig()
+
+	if cfg.BatchSize != 500 {
+		t.Fatalf("BatchSize = %d, want 500", cfg.BatchSize)
+	}
+	if cfg.BatchWait != 250 {
+		t.Fatalf("BatchWait = %d, want 250", cfg.BatchWait)
+	}
 }
 
 type contextCheckingHistoryAppender struct {
-	ctxErr error
-	count  int
+	ctxErr     error
+	count      int
+	batchCount int
+	batch      []envelope.FrameEnvelope
+	err        error
 }
 
 func (a *contextCheckingHistoryAppender) AppendAll(ctx context.Context, _ envelope.FrameEnvelope) error {
 	a.ctxErr = ctx.Err()
 	a.count++
-	return a.ctxErr
+	if a.ctxErr != nil {
+		return a.ctxErr
+	}
+	return a.err
+}
+
+func (a *contextCheckingHistoryAppender) AppendAllBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
+	a.ctxErr = ctx.Err()
+	a.batchCount++
+	a.batch = append([]envelope.FrameEnvelope(nil), envs...)
+	if a.ctxErr != nil {
+		return a.ctxErr
+	}
+	return a.err
 }
 
 type contextCheckingHistoryCommitter struct {
-	ctxErr error
-	count  int
+	ctxErr       error
+	count        int
+	messageCount int
 }
 
-func (c *contextCheckingHistoryCommitter) CommitMessages(ctx context.Context, _ ...kafka.Message) error {
+func (c *contextCheckingHistoryCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
 	c.ctxErr = ctx.Err()
 	c.count++
+	c.messageCount += len(messages)
 	return c.ctxErr
 }
 
@@ -110,3 +232,5 @@ type discardHistoryLogger struct{}
 
 func (discardHistoryLogger) Error(string, ...any) {}
 func (discardHistoryLogger) Warn(string, ...any)  {}
+
+var errTestHistoryAppend = errors.New("test history append failed")