fix(go): batch durable kafka replay

This commit is contained in:
lingniu
2026-07-02 14:01:50 +08:00
parent c939cc6b0c
commit 5a958616a3
5 changed files with 176 additions and 5 deletions

View File

@@ -139,6 +139,34 @@ func TestDurableSinkReplayLoopTickRespectsBatchSize(t *testing.T) {
}
}
func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) {
dir := t.TempDir()
env := durableTestEnvelope()
writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
writeDurableRecord(t, filepath.Join(dir, "0002-unified.json"), durableRecord{Kind: "unified", Envelope: env})
delegate := &batchRecordingSink{}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
if delegate.batchCalls != 1 {
t.Fatalf("batch calls = %d, want 1", delegate.batchCalls)
}
if delegate.rawCalls != 0 || delegate.unifiedCalls != 0 {
t.Fatalf("individual publish should not be used, raw=%d unified=%d", delegate.rawCalls, delegate.unifiedCalls)
}
if got, want := len(delegate.records), 2; got != want {
t.Fatalf("batch record count = %d, want %d", got, want)
}
if delegate.records[0].Kind != "raw" || delegate.records[1].Kind != "unified" {
t.Fatalf("batch order = %#v", delegate.records)
}
if files := spoolFiles(t, dir); len(files) != 0 {
t.Fatalf("spool files after replay = %#v, want none", files)
}
}
func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
reader := &fakeNameReader{
batches: [][]string{
@@ -298,3 +326,30 @@ func (s *contextCheckingReplaySink) PublishUnified(ctx context.Context, _ envelo
func (s *contextCheckingReplaySink) Close() error {
return nil
}
type batchRecordingSink struct {
batchCalls int
records []durableRecord
rawCalls int
unifiedCalls int
}
func (s *batchRecordingSink) PublishRecords(_ context.Context, records []durableRecord) error {
s.batchCalls++
s.records = append(s.records, records...)
return nil
}
func (s *batchRecordingSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
s.rawCalls++
return nil
}
func (s *batchRecordingSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
s.unifiedCalls++
return nil
}
func (s *batchRecordingSink) Close() error {
return nil
}