fix(go): batch durable kafka replay
This commit is contained in:
@@ -76,6 +76,25 @@ func (s *KafkaSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelo
|
||||
return s.publish(ctx, s.unifiedTopic, env)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) PublishRecords(ctx context.Context, records []durableRecord) error {
|
||||
messages := make([]kafka.Message, 0, len(records))
|
||||
for _, record := range records {
|
||||
topic, err := s.topicForRecord(record)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
message, err := kafkaMessage(topic, record.Envelope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
messages = append(messages, message)
|
||||
}
|
||||
if len(messages) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.writer.WriteMessages(ctx, messages...)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) Close() error {
|
||||
if s == nil || s.writer == nil {
|
||||
return nil
|
||||
@@ -84,13 +103,36 @@ func (s *KafkaSink) Close() error {
|
||||
}
|
||||
|
||||
func (s *KafkaSink) publish(ctx context.Context, topic string, env envelope.FrameEnvelope) error {
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
message, err := kafkaMessage(topic, env)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.writer.WriteMessages(ctx, kafka.Message{
|
||||
return s.writer.WriteMessages(ctx, message)
|
||||
}
|
||||
|
||||
func (s *KafkaSink) topicForRecord(record durableRecord) (string, error) {
|
||||
switch record.Kind {
|
||||
case "raw":
|
||||
topic, ok := s.rawTopics[record.Envelope.Protocol]
|
||||
if !ok || topic == "" {
|
||||
return "", fmt.Errorf("raw topic not configured for protocol %s", record.Envelope.Protocol)
|
||||
}
|
||||
return topic, nil
|
||||
case "unified":
|
||||
return s.unifiedTopic, nil
|
||||
default:
|
||||
return "", fmt.Errorf("unknown durable record kind %q", record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func kafkaMessage(topic string, env envelope.FrameEnvelope) (kafka.Message, error) {
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
return kafka.Message{}, err
|
||||
}
|
||||
return kafka.Message{
|
||||
Topic: topic,
|
||||
Key: env.KafkaKey(),
|
||||
Value: payload,
|
||||
})
|
||||
}, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user