lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go

package eventbus

import (
	"context"
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)

func TestDurableOutboxPersistsBeforeAsyncSubmitAndDeletesAfterAck(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{}
	var sink *DurableOutboxSink
	delegate.onSubmit = func(record durableRecord) {
		if got := outboxBacklog(sink); got != 1 {
			t.Fatalf("durable backlog visible at submit = %d, want 1", got)
		}
		if record.Envelope.EventID == "" {
			t.Fatal("submitted record must have a stable event id")
		}
	}
	var err error
	sink, err = NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	if got := outboxBacklog(sink); got != 1 {
		t.Fatalf("backlog before ack = %d, want 1", got)
	}
	delegate.completeNext(t, nil)
	if got := outboxBacklog(sink); got != 0 {
		t.Fatalf("backlog after ack = %d, want 0", got)
	}
}

func TestDurableOutboxImmediateSubmitErrorKeepsAcceptedRecord(t *testing.T) {
	dir := t.TempDir()
	submitErr := errors.New("nats pending limit")
	delegate := &outboxAsyncSink{submitErrors: []error{submitErr}}
	var reported error
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
		Directory: dir,
		OnError: func(err error) {
			reported = err
		},
	})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("accepted durable record should hide submit error, got %v", err)
	}
	if got := outboxBacklog(sink); got != 1 {
		t.Fatalf("backlog after submit error = %d, want 1", got)
	}
	if reported == nil || !strings.Contains(reported.Error(), submitErr.Error()) {
		t.Fatalf("reported error = %v", reported)
	}
}

func TestDurableOutboxAckErrorIsRetriedAndStableRecordIsRemoved(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	delegate.completeNext(t, errors.New("ack timeout"))
	if got := outboxBacklog(sink); got != 1 {
		t.Fatalf("backlog after ack error = %d, want 1", got)
	}
	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	if got := delegate.recordCount(); got != 2 {
		t.Fatalf("async submissions = %d, want 2", got)
	}
	first, second := delegate.recordAt(0), delegate.recordAt(1)
	if first.Envelope.EventID == "" || first.Envelope.EventID != second.Envelope.EventID {
		t.Fatalf("replay event ids = %q and %q", first.Envelope.EventID, second.Envelope.EventID)
	}
	delegate.completeNext(t, nil)
	if got := outboxBacklog(sink); got != 0 {
		t.Fatalf("backlog after replay ack = %d, want 0", got)
	}
}

func TestDurableOutboxReplaysPreexistingRecordAfterRestart(t *testing.T) {
	dir := t.TempDir()
	first := &outboxAsyncSink{submitErrors: []error{errors.New("nats unavailable")}}
	writer, err := NewDurableOutboxSink(first, DurableOutboxConfig{Directory: dir})
	if err != nil {
		t.Fatalf("create first outbox: %v", err)
	}
	if err := writer.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("persist restart record: %v", err)
	}
	if err := writer.Close(); err != nil {
		t.Fatalf("close first outbox: %v", err)
	}
	delegate := &outboxAsyncSink{}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir, ReplayBatchSize: 10})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	if got := delegate.recordCount(); got != 1 {
		t.Fatalf("replayed records = %d, want 1", got)
	}
	delegate.completeNext(t, nil)
	if got := outboxBacklog(sink); got != 0 {
		t.Fatalf("backlog after replay ack = %d, want 0", got)
	}
}

func TestDurableOutboxDoesNotResubmitInflightRecord(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	if got := delegate.recordCount(); got != 1 {
		t.Fatalf("submissions while inflight = %d, want 1", got)
	}
	delegate.completeNext(t, nil)
}

func TestDurableOutboxRejectsInvalidRecordBeforePersistence(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{validateErr: errors.New("subject not configured")}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}

	err = sink.PublishRaw(context.Background(), durableTestEnvelope())
	if err == nil || !strings.Contains(err.Error(), "subject not configured") {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	if got := outboxBacklog(sink); got != 0 {
		t.Fatalf("invalid record backlog = %d, want 0", got)
	}
	if got := delegate.recordCount(); got != 0 {
		t.Fatalf("invalid record submissions = %d, want 0", got)
	}
}

func TestDurableOutboxCloseWaitsForPendingAck(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
		Directory:    dir,
		CloseTimeout: time.Second,
	})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}
	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}

	done := make(chan error, 1)
	go func() { done <- sink.Close() }()
	select {
	case err := <-done:
		t.Fatalf("Close() returned before ack: %v", err)
	case <-time.After(20 * time.Millisecond):
	}
	delegate.completeNext(t, nil)
	select {
	case err := <-done:
		if err != nil {
			t.Fatalf("Close() error = %v", err)
		}
	case <-time.After(time.Second):
		t.Fatal("Close() did not return after ack")
	}
}

func TestDurableOutboxCloseTimeoutRetainsRecord(t *testing.T) {
	dir := t.TempDir()
	delegate := &outboxAsyncSink{}
	sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
		Directory:    dir,
		CloseTimeout: 10 * time.Millisecond,
	})
	if err != nil {
		t.Fatalf("NewDurableOutboxSink() error = %v", err)
	}
	if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}

	err = sink.Close()
	if err == nil || !strings.Contains(err.Error(), "close timed out") {
		t.Fatalf("Close() error = %v", err)
	}
	if got := outboxBacklog(sink); got != 1 {
		t.Fatalf("backlog after close timeout = %d, want 1", got)
	}
	delegate.completeNext(t, errors.New("connection closed"))
}

func BenchmarkDurableOutboxPublishRaw(b *testing.B) {
	for _, syncWrites := range []bool{false, true} {
		name := "fsync_off"
		if syncWrites {
			name = "fsync_on"
		}
		b.Run(name, func(b *testing.B) {
			sink, err := NewDurableOutboxSink(autoAckOutboxSink{}, DurableOutboxConfig{
				Directory:  b.TempDir(),
				SyncWrites: syncWrites,
			})
			if err != nil {
				b.Fatalf("NewDurableOutboxSink() error = %v", err)
			}
			var sequence atomic.Uint32
			b.ResetTimer()
			b.RunParallel(func(pb *testing.PB) {
				for pb.Next() {
					env := durableTestEnvelope()
					env.EventID = ""
					env.Sequence = uint16(sequence.Add(1))
					if err := sink.PublishRaw(context.Background(), env); err != nil {
						b.Fatalf("PublishRaw() error = %v", err)
					}
				}
			})
			b.StopTimer()
			if err := sink.Close(); err != nil {
				b.Fatalf("Close() error = %v", err)
			}
		})
	}
}

func outboxBacklog(sink *DurableOutboxSink) int {
	count, _ := sink.store.Stats()
	return count
}

type outboxAsyncSink struct {
	mu           sync.Mutex
	validateErr  error
	submitErrors []error
	records      []durableRecord
	callbacks    []func(error)
	onSubmit     func(durableRecord)
	closeCalls   int
}

func (s *outboxAsyncSink) ValidateRecord(record durableRecord) error {
	if s.validateErr != nil {
		return s.validateErr
	}
	switch record.Kind {
	case "raw", "unified", "fields":
		return nil
	default:
		return errUnknownRecordKind(record.Kind)
	}
}

func (s *outboxAsyncSink) PublishRecordAsync(record durableRecord, complete func(error)) error {
	if s.onSubmit != nil {
		s.onSubmit(record)
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if len(s.submitErrors) > 0 {
		err := s.submitErrors[0]
		s.submitErrors = s.submitErrors[1:]
		if err != nil {
			return err
		}
	}
	s.records = append(s.records, record)
	s.callbacks = append(s.callbacks, complete)
	return nil
}

func (s *outboxAsyncSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (s *outboxAsyncSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (s *outboxAsyncSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (s *outboxAsyncSink) Close() error {
	s.mu.Lock()
	s.closeCalls++
	s.mu.Unlock()
	return nil
}

func (s *outboxAsyncSink) completeNext(t *testing.T, err error) {
	t.Helper()
	s.mu.Lock()
	if len(s.callbacks) == 0 {
		s.mu.Unlock()
		t.Fatal("no pending async callback")
	}
	callback := s.callbacks[0]
	s.callbacks = s.callbacks[1:]
	s.mu.Unlock()
	callback(err)
}

func (s *outboxAsyncSink) recordCount() int {
	s.mu.Lock()
	defer s.mu.Unlock()
	return len(s.records)
}

func (s *outboxAsyncSink) recordAt(index int) durableRecord {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.records[index]
}

type autoAckOutboxSink struct{}

func (autoAckOutboxSink) ValidateRecord(record durableRecord) error {
	switch record.Kind {
	case "raw", "unified", "fields":
		return nil
	default:
		return errUnknownRecordKind(record.Kind)
	}
}

func (autoAckOutboxSink) PublishRecordAsync(_ durableRecord, complete func(error)) error {
	complete(nil)
	return nil
}

func (autoAckOutboxSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (autoAckOutboxSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (autoAckOutboxSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (autoAckOutboxSink) Close() error { return nil }