fix(go): stream durable spool batches

This commit is contained in:
lingniu
2026-07-02 13:47:53 +08:00
parent b27f909109
commit c939cc6b0c
2 changed files with 98 additions and 6 deletions

View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"testing"
@@ -138,6 +139,32 @@ func TestDurableSinkReplayLoopTickRespectsBatchSize(t *testing.T) {
}
}
func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
reader := &fakeNameReader{
batches: [][]string{
{"0001-raw.json", "notes.txt", "0002-unified.json"},
{"0003-raw.json"},
},
}
files, err := durableFilesFromReader("/spool", 2, reader)
if err != nil {
t.Fatalf("durableFilesFromReader() error = %v", err)
}
if reader.calls != 1 {
t.Fatalf("Readdirnames calls = %d, want 1", reader.calls)
}
want := []string{filepath.Join("/spool", "0001-raw.json"), filepath.Join("/spool", "0002-unified.json")}
if len(files) != len(want) {
t.Fatalf("files = %#v, want %#v", files, want)
}
for i := range want {
if files[i] != want[i] {
t.Fatalf("files[%d] = %q, want %q; all=%#v", i, files[i], want[i], files)
}
}
}
func durableTestEnvelope() envelope.FrameEnvelope {
return envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
@@ -151,6 +178,24 @@ func durableTestEnvelope() envelope.FrameEnvelope {
}
}
type fakeNameReader struct {
batches [][]string
calls int
}
func (r *fakeNameReader) Readdirnames(int) ([]string, error) {
r.calls++
if len(r.batches) == 0 {
return nil, io.EOF
}
batch := r.batches[0]
r.batches = r.batches[1:]
if len(r.batches) == 0 {
return batch, io.EOF
}
return batch, nil
}
func writeDurableRecord(t *testing.T, path string, record durableRecord) {
t.Helper()
payload, err := json.Marshal(record)