fix(go): batch durable kafka replay

This commit is contained in:
lingniu
2026-07-02 14:01:50 +08:00
parent c939cc6b0c
commit 5a958616a3
5 changed files with 176 additions and 5 deletions

View File

@@ -40,6 +40,10 @@ type durableRecordFile struct {
record durableRecord
}
type recordPublishingSink interface {
PublishRecords(context.Context, []durableRecord) error
}
const durableReplayOperationTimeout = 30 * time.Second
func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
@@ -93,6 +97,24 @@ func (s *DurableSink) replay(ctx context.Context, limit int) error {
records = append(records, durableRecordFile{path: file, record: record})
}
sortDurableRecords(records)
if publisher, ok := s.delegate.(recordPublishingSink); ok {
durableRecords := make([]durableRecord, 0, len(records))
for _, item := range records {
durableRecords = append(durableRecords, item.record)
}
if err := publisher.PublishRecords(ctx, durableRecords); err != nil {
return err
}
for _, item := range records {
if err := os.Remove(item.path); err != nil {
return err
}
if item.record.Kind == "raw" {
s.clearRawPending(item.record.Envelope)
}
}
return nil
}
for _, item := range records {
if err := s.publishRecord(ctx, item.record); err != nil {
return err
@@ -163,10 +185,14 @@ func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) e
case "unified":
return s.delegate.PublishUnified(ctx, record.Envelope)
default:
return fmt.Errorf("unknown durable record kind %q", record.Kind)
return errUnknownRecordKind(record.Kind)
}
}
func errUnknownRecordKind(kind string) error {
return fmt.Errorf("unknown durable record kind %q", kind)
}
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
if s.dir == "" {
return fmt.Errorf("durable spool directory is empty")