fix(go): batch durable kafka replay
This commit is contained in:
@@ -139,6 +139,34 @@ func TestDurableSinkReplayLoopTickRespectsBatchSize(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
env := durableTestEnvelope()
|
||||
writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
|
||||
writeDurableRecord(t, filepath.Join(dir, "0002-unified.json"), durableRecord{Kind: "unified", Envelope: env})
|
||||
delegate := &batchRecordingSink{}
|
||||
sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
|
||||
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
if delegate.batchCalls != 1 {
|
||||
t.Fatalf("batch calls = %d, want 1", delegate.batchCalls)
|
||||
}
|
||||
if delegate.rawCalls != 0 || delegate.unifiedCalls != 0 {
|
||||
t.Fatalf("individual publish should not be used, raw=%d unified=%d", delegate.rawCalls, delegate.unifiedCalls)
|
||||
}
|
||||
if got, want := len(delegate.records), 2; got != want {
|
||||
t.Fatalf("batch record count = %d, want %d", got, want)
|
||||
}
|
||||
if delegate.records[0].Kind != "raw" || delegate.records[1].Kind != "unified" {
|
||||
t.Fatalf("batch order = %#v", delegate.records)
|
||||
}
|
||||
if files := spoolFiles(t, dir); len(files) != 0 {
|
||||
t.Fatalf("spool files after replay = %#v, want none", files)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
|
||||
reader := &fakeNameReader{
|
||||
batches: [][]string{
|
||||
@@ -298,3 +326,30 @@ func (s *contextCheckingReplaySink) PublishUnified(ctx context.Context, _ envelo
|
||||
func (s *contextCheckingReplaySink) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
type batchRecordingSink struct {
|
||||
batchCalls int
|
||||
records []durableRecord
|
||||
rawCalls int
|
||||
unifiedCalls int
|
||||
}
|
||||
|
||||
func (s *batchRecordingSink) PublishRecords(_ context.Context, records []durableRecord) error {
|
||||
s.batchCalls++
|
||||
s.records = append(s.records, records...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *batchRecordingSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
s.rawCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *batchRecordingSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
s.unifiedCalls++
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *batchRecordingSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user