shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(go): protect kafka message processing during shutdown

Browse Source
This commit is contained in:
lingniu
2026-07-02 13:20:46 +08:00
parent a80eb2eab4
commit 640f70636d
6 changed files with 311 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
go/vehicle-gateway/cmd/history-writer/main.go
View File

@@ -8,6 +8,7 @@ import (
"os/signal"
"strings"
"syscall"
"time"
"github.com/segmentio/kafka-go"
_ "github.com/taosdata/driver-go/v3/taosWS"
@@ -65,19 +66,39 @@ func main() {
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.AppendAll(ctx, env); err != nil {
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
processHistoryMessage(ctx, logger, writer, reader, message)
}
}
const kafkaMessageOperationTimeout = 30 * time.Second
type historyAppender interface {
AppendAll(context.Context, envelope.FrameEnvelope) error
}
type kafkaMessageCommitter interface {
CommitMessages(context.Context, ...kafka.Message) error
}
func processHistoryMessage(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, appender historyAppender, committer kafkaMessageCommitter, message kafka.Message) {
messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
defer cancel()
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = committer.CommitMessages(messageCtx, message)
return
}
if err := appender.AppendAll(messageCtx, env); err != nil {
logger.Error("tdengine append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
return
}
if err := committer.CommitMessages(messageCtx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}