shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(go): protect kafka message processing during shutdown

Browse Source
This commit is contained in:
lingniu
2026-07-02 13:20:46 +08:00
parent a80eb2eab4
commit 640f70636d
6 changed files with 311 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
go/vehicle-gateway/cmd/stat-writer/main.go
View File

@@ -62,19 +62,39 @@ func main() {
logger.Error("kafka fetch failed", "error", err)
continue
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = reader.CommitMessages(ctx, message)
continue
}
if err := writer.Append(ctx, env); err != nil {
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
continue
}
if err := reader.CommitMessages(ctx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
processStatMessage(ctx, logger, writer, reader, message)
}
}
const kafkaMessageOperationTimeout = 30 * time.Second
type statAppender interface {
Append(context.Context, envelope.FrameEnvelope) error
}
type kafkaMessageCommitter interface {
CommitMessages(context.Context, ...kafka.Message) error
}
func processStatMessage(ctx context.Context, logger interface {
Error(string, ...any)
Warn(string, ...any)
}, appender statAppender, committer kafkaMessageCommitter, message kafka.Message) {
messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
defer cancel()
var env envelope.FrameEnvelope
if err := json.Unmarshal(message.Value, &env); err != nil {
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
_ = committer.CommitMessages(messageCtx, message)
return
}
if err := appender.Append(messageCtx, env); err != nil {
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
return
}
if err := committer.CommitMessages(messageCtx, message); err != nil {
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
}
}