fix(go): batch durable kafka replay

This commit is contained in:
lingniu
2026-07-02 14:01:50 +08:00
parent c939cc6b0c
commit 5a958616a3
5 changed files with 176 additions and 5 deletions

View File

@@ -76,6 +76,25 @@ func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelo
return s.publish(ctx, s.unifiedTopic, env)
}
func (s *KafkaSink) PublishRecords(ctx context.Context, records []durableRecord) error {
messages := make([]kafka.Message, 0, len(records))
for _, record := range records {
topic, err := s.topicForRecord(record)
if err != nil {
return err
}
message, err := kafkaMessage(topic, record.Envelope)
if err != nil {
return err
}
messages = append(messages, message)
}
if len(messages) == 0 {
return nil
}
return s.writer.WriteMessages(ctx, messages...)
}
func (s *KafkaSink) Close() error {
if s == nil || s.writer == nil {
return nil
@@ -84,13 +103,36 @@ func (s *KafkaSink) Close() error {
}
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
payload, err := env.MarshalJSONBytes()
message, err := kafkaMessage(topic, env)
if err != nil {
return err
}
return s.writer.WriteMessages(ctx, kafka.Message{
return s.writer.WriteMessages(ctx, message)
}
func (s *KafkaSink) topicForRecord(record durableRecord) (string, error) {
switch record.Kind {
case "raw":
topic, ok := s.rawTopics[record.Envelope.Protocol]
if !ok || topic == "" {
return "", fmt.Errorf("raw topic not configured for protocol %s", record.Envelope.Protocol)
}
return topic, nil
case "unified":
return s.unifiedTopic, nil
default:
return "", fmt.Errorf("unknown durable record kind %q", record.Kind)
}
}
func kafkaMessage(topic string, env envelope.FrameEnvelope) (kafka.Message, error) {
payload, err := env.MarshalJSONBytes()
if err != nil {
return kafka.Message{}, err
}
return kafka.Message{
Topic: topic,
Key: env.KafkaKey(),
Value: payload,
})
}, nil
}