fix(go): batch durable kafka replay
This commit is contained in:
@@ -40,6 +40,10 @@ type durableRecordFile struct {
|
||||
record durableRecord
|
||||
}
|
||||
|
||||
type recordPublishingSink interface {
|
||||
PublishRecords(context.Context, []durableRecord) error
|
||||
}
|
||||
|
||||
const durableReplayOperationTimeout = 30 * time.Second
|
||||
|
||||
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
|
||||
@@ -93,6 +97,24 @@ func (s *DurableSink) replay(ctx context.Context, limit int) error {
|
||||
records = append(records, durableRecordFile{path: file, record: record})
|
||||
}
|
||||
sortDurableRecords(records)
|
||||
if publisher, ok := s.delegate.(recordPublishingSink); ok {
|
||||
durableRecords := make([]durableRecord, 0, len(records))
|
||||
for _, item := range records {
|
||||
durableRecords = append(durableRecords, item.record)
|
||||
}
|
||||
if err := publisher.PublishRecords(ctx, durableRecords); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range records {
|
||||
if err := os.Remove(item.path); err != nil {
|
||||
return err
|
||||
}
|
||||
if item.record.Kind == "raw" {
|
||||
s.clearRawPending(item.record.Envelope)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, item := range records {
|
||||
if err := s.publishRecord(ctx, item.record); err != nil {
|
||||
return err
|
||||
@@ -163,10 +185,14 @@ func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) e
|
||||
case "unified":
|
||||
return s.delegate.PublishUnified(ctx, record.Envelope)
|
||||
default:
|
||||
return fmt.Errorf("unknown durable record kind %q", record.Kind)
|
||||
return errUnknownRecordKind(record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func errUnknownRecordKind(kind string) error {
|
||||
return fmt.Errorf("unknown durable record kind %q", kind)
|
||||
}
|
||||
|
||||
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
|
||||
if s.dir == "" {
|
||||
return fmt.Errorf("durable spool directory is empty")
|
||||
|
||||
Reference in New Issue
Block a user