package eventbus import ( "context" "errors" "strings" "sync" "sync/atomic" "testing" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" ) func TestDurableOutboxPersistsBeforeAsyncSubmitAndDeletesAfterAck(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{} var sink *DurableOutboxSink delegate.onSubmit = func(record durableRecord) { if got := outboxBacklog(sink); got != 1 { t.Fatalf("durable backlog visible at submit = %d, want 1", got) } if record.Envelope.EventID == "" { t.Fatal("submitted record must have a stable event id") } } var err error sink, err = NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("PublishRaw() error = %v", err) } if got := outboxBacklog(sink); got != 1 { t.Fatalf("backlog before ack = %d, want 1", got) } delegate.completeNext(t, nil) if got := outboxBacklog(sink); got != 0 { t.Fatalf("backlog after ack = %d, want 0", got) } } func TestDurableOutboxImmediateSubmitErrorKeepsAcceptedRecord(t *testing.T) { dir := t.TempDir() submitErr := errors.New("nats pending limit") delegate := &outboxAsyncSink{submitErrors: []error{submitErr}} var reported error sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ Directory: dir, OnError: func(err error) { reported = err }, }) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("accepted durable record should hide submit error, got %v", err) } if got := outboxBacklog(sink); got != 1 { t.Fatalf("backlog after submit error = %d, want 1", got) } if reported == nil || !strings.Contains(reported.Error(), submitErr.Error()) { t.Fatalf("reported error = %v", reported) } } func TestDurableOutboxAckErrorIsRetriedAndStableRecordIsRemoved(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("PublishRaw() error = %v", err) } delegate.completeNext(t, errors.New("ack timeout")) if got := outboxBacklog(sink); got != 1 { t.Fatalf("backlog after ack error = %d, want 1", got) } if err := sink.ReplayOnce(context.Background()); err != nil { t.Fatalf("ReplayOnce() error = %v", err) } if got := delegate.recordCount(); got != 2 { t.Fatalf("async submissions = %d, want 2", got) } first, second := delegate.recordAt(0), delegate.recordAt(1) if first.Envelope.EventID == "" || first.Envelope.EventID != second.Envelope.EventID { t.Fatalf("replay event ids = %q and %q", first.Envelope.EventID, second.Envelope.EventID) } delegate.completeNext(t, nil) if got := outboxBacklog(sink); got != 0 { t.Fatalf("backlog after replay ack = %d, want 0", got) } } func TestDurableOutboxReplaysPreexistingRecordAfterRestart(t *testing.T) { dir := t.TempDir() first := &outboxAsyncSink{submitErrors: []error{errors.New("nats unavailable")}} writer, err := NewDurableOutboxSink(first, DurableOutboxConfig{Directory: dir}) if err != nil { t.Fatalf("create first outbox: %v", err) } if err := writer.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("persist restart record: %v", err) } if err := writer.Close(); err != nil { t.Fatalf("close first outbox: %v", err) } delegate := &outboxAsyncSink{} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir, ReplayBatchSize: 10}) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.ReplayOnce(context.Background()); err != nil { t.Fatalf("ReplayOnce() error = %v", err) } if got := delegate.recordCount(); got != 1 { t.Fatalf("replayed records = %d, want 1", got) } delegate.completeNext(t, nil) if got := outboxBacklog(sink); got != 0 { t.Fatalf("backlog after replay ack = %d, want 0", got) } } func TestDurableOutboxDoesNotResubmitInflightRecord(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("PublishRaw() error = %v", err) } if err := sink.ReplayOnce(context.Background()); err != nil { t.Fatalf("ReplayOnce() error = %v", err) } if got := delegate.recordCount(); got != 1 { t.Fatalf("submissions while inflight = %d, want 1", got) } delegate.completeNext(t, nil) } func TestDurableOutboxRejectsInvalidRecordBeforePersistence(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{validateErr: errors.New("subject not configured")} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir}) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } err = sink.PublishRaw(context.Background(), durableTestEnvelope()) if err == nil || !strings.Contains(err.Error(), "subject not configured") { t.Fatalf("PublishRaw() error = %v", err) } if got := outboxBacklog(sink); got != 0 { t.Fatalf("invalid record backlog = %d, want 0", got) } if got := delegate.recordCount(); got != 0 { t.Fatalf("invalid record submissions = %d, want 0", got) } } func TestDurableOutboxCloseWaitsForPendingAck(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ Directory: dir, CloseTimeout: time.Second, }) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("PublishRaw() error = %v", err) } done := make(chan error, 1) go func() { done <- sink.Close() }() select { case err := <-done: t.Fatalf("Close() returned before ack: %v", err) case <-time.After(20 * time.Millisecond): } delegate.completeNext(t, nil) select { case err := <-done: if err != nil { t.Fatalf("Close() error = %v", err) } case <-time.After(time.Second): t.Fatal("Close() did not return after ack") } } func TestDurableOutboxCloseTimeoutRetainsRecord(t *testing.T) { dir := t.TempDir() delegate := &outboxAsyncSink{} sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{ Directory: dir, CloseTimeout: 10 * time.Millisecond, }) if err != nil { t.Fatalf("NewDurableOutboxSink() error = %v", err) } if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil { t.Fatalf("PublishRaw() error = %v", err) } err = sink.Close() if err == nil || !strings.Contains(err.Error(), "close timed out") { t.Fatalf("Close() error = %v", err) } if got := outboxBacklog(sink); got != 1 { t.Fatalf("backlog after close timeout = %d, want 1", got) } delegate.completeNext(t, errors.New("connection closed")) } func BenchmarkDurableOutboxPublishRaw(b *testing.B) { for _, syncWrites := range []bool{false, true} { name := "fsync_off" if syncWrites { name = "fsync_on" } b.Run(name, func(b *testing.B) { sink, err := NewDurableOutboxSink(autoAckOutboxSink{}, DurableOutboxConfig{ Directory: b.TempDir(), SyncWrites: syncWrites, }) if err != nil { b.Fatalf("NewDurableOutboxSink() error = %v", err) } var sequence atomic.Uint32 b.ResetTimer() b.RunParallel(func(pb *testing.PB) { for pb.Next() { env := durableTestEnvelope() env.EventID = "" env.Sequence = uint16(sequence.Add(1)) if err := sink.PublishRaw(context.Background(), env); err != nil { b.Fatalf("PublishRaw() error = %v", err) } } }) b.StopTimer() if err := sink.Close(); err != nil { b.Fatalf("Close() error = %v", err) } }) } } func outboxBacklog(sink *DurableOutboxSink) int { count, _ := sink.store.Stats() return count } type outboxAsyncSink struct { mu sync.Mutex validateErr error submitErrors []error records []durableRecord callbacks []func(error) onSubmit func(durableRecord) closeCalls int } func (s *outboxAsyncSink) ValidateRecord(record durableRecord) error { if s.validateErr != nil { return s.validateErr } switch record.Kind { case "raw", "unified", "fields": return nil default: return errUnknownRecordKind(record.Kind) } } func (s *outboxAsyncSink) PublishRecordAsync(record durableRecord, complete func(error)) error { if s.onSubmit != nil { s.onSubmit(record) } s.mu.Lock() defer s.mu.Unlock() if len(s.submitErrors) > 0 { err := s.submitErrors[0] s.submitErrors = s.submitErrors[1:] if err != nil { return err } } s.records = append(s.records, record) s.callbacks = append(s.callbacks, complete) return nil } func (s *outboxAsyncSink) PublishRaw(context.Context, envelope.FrameEnvelope) error { return nil } func (s *outboxAsyncSink) PublishUnified(context.Context, envelope.FrameEnvelope) error { return nil } func (s *outboxAsyncSink) PublishFields(context.Context, envelope.FrameEnvelope) error { return nil } func (s *outboxAsyncSink) Close() error { s.mu.Lock() s.closeCalls++ s.mu.Unlock() return nil } func (s *outboxAsyncSink) completeNext(t *testing.T, err error) { t.Helper() s.mu.Lock() if len(s.callbacks) == 0 { s.mu.Unlock() t.Fatal("no pending async callback") } callback := s.callbacks[0] s.callbacks = s.callbacks[1:] s.mu.Unlock() callback(err) } func (s *outboxAsyncSink) recordCount() int { s.mu.Lock() defer s.mu.Unlock() return len(s.records) } func (s *outboxAsyncSink) recordAt(index int) durableRecord { s.mu.Lock() defer s.mu.Unlock() return s.records[index] } type autoAckOutboxSink struct{} func (autoAckOutboxSink) ValidateRecord(record durableRecord) error { switch record.Kind { case "raw", "unified", "fields": return nil default: return errUnknownRecordKind(record.Kind) } } func (autoAckOutboxSink) PublishRecordAsync(_ durableRecord, complete func(error)) error { complete(nil) return nil } func (autoAckOutboxSink) PublishRaw(context.Context, envelope.FrameEnvelope) error { return nil } func (autoAckOutboxSink) PublishUnified(context.Context, envelope.FrameEnvelope) error { return nil } func (autoAckOutboxSink) PublishFields(context.Context, envelope.FrameEnvelope) error { return nil } func (autoAckOutboxSink) Close() error { return nil }