shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): batch tdengine history writes

Browse Source
This commit is contained in:
lingniu
2026-07-03 18:37:38 +08:00
parent 3b0c5fbd5a
commit 09f3e68891
8 changed files with 447 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
go/vehicle-gateway/cmd/history-writer/main.go
View File

@@ -6,6 +6,7 @@ import (
"encoding/json"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
@@ -62,7 +63,9 @@ func main() {
logger.Info("history writer started",
"driver", cfg.TDengineDriver,
"group", cfg.KafkaGroup,
"topics", strings.Join(cfg.KafkaTopics, ","))
"topics", strings.Join(cfg.KafkaTopics, ","),
"batch_size", cfg.BatchSize,
"batch_wait_ms", cfg.BatchWait)
for {
message, err := reader.FetchMessage(ctx)
@@ -73,7 +76,8 @@ func main() {
logger.Error("kafka fetch failed", "error", err)
continue
}
processHistoryMessage(ctx, logger, registry, writer, reader, message)
batch := collectHistoryBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond)
processHistoryBatch(ctx, logger, registry, writer, reader, batch)
}
}
@@ -81,12 +85,42 @@ const kafkaMessageOperationTimeout = 30 * time.Second
type historyAppender interface {
AppendAll(context.Context, envelope.FrameEnvelope) error
AppendAllBatch(context.Context, []envelope.FrameEnvelope) error
}
type kafkaMessageCommitter interface {
CommitMessages(context.Context, ...kafka.Message) error
}
type kafkaMessageFetcher interface {
FetchMessage(context.Context) (kafka.Message, error)
}
func collectHistoryBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
if maxSize <= 1 {
return []kafka.Message{first}
}
if maxWait <= 0 {
maxWait = 100 * time.Millisecond
}
batch := []kafka.Message{first}
deadline := time.Now().Add(maxWait)
for len(batch) < maxSize {
remaining := time.Until(deadline)
if remaining <= 0 {
break
}
fetchCtx, cancel := context.WithTimeout(ctx, remaining)
message, err := fetcher.FetchMessage(fetchCtx)
cancel()
if err != nil {
break
}
batch = append(batch, message)
}
return batch
}
func processHistoryMessage(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
@@ -117,6 +151,64 @@ func processHistoryMessage(ctx context.Context, logger interface {
addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok")
}
func processHistoryBatch(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, registry *metrics.Registry, appender historyAppender, committer kafkaMessageCommitter, messages []kafka.Message) {
if len(messages) == 0 {
return
}
messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
defer cancel()
envelopes := make([]envelope.FrameEnvelope, 0, len(messages))
for _, message := range messages {
addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, "received")
addWriterLagMetric(registry, message)
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
addWriterMetric(registry, "vehicle_history_kafka_messages_total", message, "invalid_json")
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
continue
}
envelopes = append(envelopes, env)
}
if len(envelopes) > 0 {
started := time.Now()
err := appender.AppendAllBatch(messageCtx, envelopes)
elapsed := time.Since(started)
status := "ok"
if err != nil {
status = "error"
}
addBatchMetric(registry, "vehicle_history_batch_flush_total", status, 1)
addBatchMetric(registry, "vehicle_history_batch_rows_total", status, float64(len(envelopes)))
setBatchDuration(registry, status, elapsed)
if err != nil {
for _, message := range messages {
addWriterMetric(registry, "vehicle_history_writes_total", message, "error")
}
first := messages[0]
logger.Error("tdengine batch append failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "rows", len(envelopes), "error", err)
return
}
for _, message := range messages {
addWriterMetric(registry, "vehicle_history_writes_total", message, "ok")
}
}
if err := committer.CommitMessages(messageCtx, messages...); err != nil {
for _, message := range messages {
addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "error")
}
first := messages[0]
logger.Error("kafka batch commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err)
return
}
for _, message := range messages {
addWriterMetric(registry, "vehicle_history_kafka_commits_total", message, "ok")
}
}
func addWriterMetric(registry *metrics.Registry, name string, message kafka.Message, status string) {
if registry == nil {
return
@@ -124,6 +216,20 @@ func addWriterMetric(registry *metrics.Registry, name string, message kafka.Mess
registry.IncCounter(name, metrics.Labels{"topic": message.Topic, "status": status})
}
func addBatchMetric(registry *metrics.Registry, name string, status string, value float64) {
if registry == nil {
return
}
registry.AddCounter(name, metrics.Labels{"status": status}, value)
}
func setBatchDuration(registry *metrics.Registry, status string, elapsed time.Duration) {
if registry == nil {
return
}
registry.SetGauge("vehicle_history_batch_flush_duration_ms", metrics.Labels{"status": status}, float64(elapsed.Milliseconds()))
}
func addWriterLagMetric(registry *metrics.Registry, message kafka.Message) {
if registry == nil {
return
@@ -139,6 +245,8 @@ type config struct {
TDengineDSN string
TDengineDatabase string
EnsureSchema bool
BatchSize int
BatchWait int
}
func loadConfig() config {
@@ -150,6 +258,8 @@ func loadConfig() config {
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
EnsureSchema: env("TDENGINE_ENSURE_SCHEMA", "true") != "false",
BatchSize: envInt("HISTORY_BATCH_SIZE", 200),
BatchWait: envInt("HISTORY_BATCH_WAIT_MS", 100),
}
}
@@ -161,6 +271,18 @@ func env(key string, fallback string) string {
return value
}
func envInt(key string, fallback int) int {
value := strings.TrimSpace(os.Getenv(key))
if value == "" {
return fallback
}
parsed, err := strconv.Atoi(value)
if err != nil {
return fallback
}
return parsed
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {

136
go/vehicle-gateway/cmd/history-writer/main_test.go
View File

@@ -3,6 +3,7 @@ package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
@@ -74,6 +75,89 @@ func TestProcessHistoryMessageRecordsMetrics(t *testing.T) {
}
}
func TestProcessHistoryBatchAppendsAllBeforeCommit(t *testing.T) {
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
firstPayload, err := json.Marshal(first)
if err != nil {
t.Fatal(err)
}
secondPayload, err := json.Marshal(second)
if err != nil {
t.Fatal(err)
}
appender := &contextCheckingHistoryAppender{}
committer := &contextCheckingHistoryCommitter{}
registry := metrics.NewRegistry()
processHistoryBatch(
context.Background(),
discardHistoryLogger{},
registry,
appender,
committer,
[]kafka.Message{
{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 13, Value: firstPayload},
{Topic: "vehicle.raw.go.jt808.v1", Partition: 2, Offset: 20, HighWaterMark: 21, Value: secondPayload},
},
)
if appender.batchCount != 1 {
t.Fatalf("batch appends = %d, want 1", appender.batchCount)
}
if got := len(appender.batch); got != 2 {
t.Fatalf("batch size = %d, want 2", got)
}
if committer.count != 1 {
t.Fatalf("commit calls = %d, want 1", committer.count)
}
if committer.messageCount != 2 {
t.Fatalf("committed messages = %d, want 2", committer.messageCount)
}
text := registry.Render()
for _, want := range []string{
`vehicle_history_batch_rows_total{status="ok"} 2`,
`vehicle_history_batch_flush_total{status="ok"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("batch metric missing %s:\n%s", want, text)
}
}
}
func TestProcessHistoryBatchDoesNotCommitWhenAppendFails(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
payload, err := json.Marshal(env)
if err != nil {
t.Fatal(err)
}
appender := &contextCheckingHistoryAppender{err: errTestHistoryAppend}
committer := &contextCheckingHistoryCommitter{}
registry := metrics.NewRegistry()
processHistoryBatch(
context.Background(),
discardHistoryLogger{},
registry,
appender,
committer,
[]kafka.Message{{Topic: "vehicle.raw.go.gb32960.v1", Partition: 1, Offset: 10, HighWaterMark: 11, Value: payload}},
)
if committer.count != 0 {
t.Fatalf("commit calls = %d, want 0", committer.count)
}
text := registry.Render()
for _, want := range []string{
`vehicle_history_batch_rows_total{status="error"} 1`,
`vehicle_history_batch_flush_total{status="error"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("batch error metric missing %s:\n%s", want, text)
}
}
}
func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
cfg := loadConfig()
@@ -82,27 +166,65 @@ func TestLoadConfigDefaultsToGoRawTopics(t *testing.T) {
if got != want {
t.Fatalf("KafkaTopics = %q, want %q", got, want)
}
if cfg.BatchSize != 200 {
t.Fatalf("BatchSize = %d, want 200", cfg.BatchSize)
}
if cfg.BatchWait != 100 {
t.Fatalf("BatchWait = %d, want 100", cfg.BatchWait)
}
}
func TestLoadConfigReadsBatchSettings(t *testing.T) {
t.Setenv("HISTORY_BATCH_SIZE", "500")
t.Setenv("HISTORY_BATCH_WAIT_MS", "250")
cfg := loadConfig()
if cfg.BatchSize != 500 {
t.Fatalf("BatchSize = %d, want 500", cfg.BatchSize)
}
if cfg.BatchWait != 250 {
t.Fatalf("BatchWait = %d, want 250", cfg.BatchWait)
}
}
type contextCheckingHistoryAppender struct {
ctxErr error
count int
ctxErr error
count int
batchCount int
batch []envelope.FrameEnvelope
err error
}
func (a *contextCheckingHistoryAppender) AppendAll(ctx context.Context, _ envelope.FrameEnvelope) error {
a.ctxErr = ctx.Err()
a.count++
return a.ctxErr
if a.ctxErr != nil {
return a.ctxErr
}
return a.err
}
func (a *contextCheckingHistoryAppender) AppendAllBatch(ctx context.Context, envs []envelope.FrameEnvelope) error {
a.ctxErr = ctx.Err()
a.batchCount++
a.batch = append([]envelope.FrameEnvelope(nil), envs...)
if a.ctxErr != nil {
return a.ctxErr
}
return a.err
}
type contextCheckingHistoryCommitter struct {
ctxErr error
count int
ctxErr error
count int
messageCount int
}
func (c *contextCheckingHistoryCommitter) CommitMessages(ctx context.Context, _ ...kafka.Message) error {
func (c *contextCheckingHistoryCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
c.ctxErr = ctx.Err()
c.count++
c.messageCount += len(messages)
return c.ctxErr
}
@@ -110,3 +232,5 @@ type discardHistoryLogger struct{}
func (discardHistoryLogger) Error(string, ...any) {}
func (discardHistoryLogger) Warn(string, ...any) {}
var errTestHistoryAppend = errors.New("test history append failed")