feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
380
go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go
Normal file
380
go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go
Normal file
@@ -0,0 +1,380 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
)
|
||||
|
||||
func TestDurableOutboxPersistsBeforeAsyncSubmitAndDeletesAfterAck(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{}
|
||||
var sink *DurableOutboxSink
|
||||
delegate.onSubmit = func(record durableRecord) {
|
||||
if got := outboxBacklog(sink); got != 1 {
|
||||
t.Fatalf("durable backlog visible at submit = %d, want 1", got)
|
||||
}
|
||||
if record.Envelope.EventID == "" {
|
||||
t.Fatal("submitted record must have a stable event id")
|
||||
}
|
||||
}
|
||||
var err error
|
||||
sink, err = NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if got := outboxBacklog(sink); got != 1 {
|
||||
t.Fatalf("backlog before ack = %d, want 1", got)
|
||||
}
|
||||
delegate.completeNext(t, nil)
|
||||
if got := outboxBacklog(sink); got != 0 {
|
||||
t.Fatalf("backlog after ack = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxImmediateSubmitErrorKeepsAcceptedRecord(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
submitErr := errors.New("nats pending limit")
|
||||
delegate := &outboxAsyncSink{submitErrors: []error{submitErr}}
|
||||
var reported error
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
|
||||
Directory: dir,
|
||||
OnError: func(err error) {
|
||||
reported = err
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("accepted durable record should hide submit error, got %v", err)
|
||||
}
|
||||
if got := outboxBacklog(sink); got != 1 {
|
||||
t.Fatalf("backlog after submit error = %d, want 1", got)
|
||||
}
|
||||
if reported == nil || !strings.Contains(reported.Error(), submitErr.Error()) {
|
||||
t.Fatalf("reported error = %v", reported)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxAckErrorIsRetriedAndStableRecordIsRemoved(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
delegate.completeNext(t, errors.New("ack timeout"))
|
||||
if got := outboxBacklog(sink); got != 1 {
|
||||
t.Fatalf("backlog after ack error = %d, want 1", got)
|
||||
}
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
if got := delegate.recordCount(); got != 2 {
|
||||
t.Fatalf("async submissions = %d, want 2", got)
|
||||
}
|
||||
first, second := delegate.recordAt(0), delegate.recordAt(1)
|
||||
if first.Envelope.EventID == "" || first.Envelope.EventID != second.Envelope.EventID {
|
||||
t.Fatalf("replay event ids = %q and %q", first.Envelope.EventID, second.Envelope.EventID)
|
||||
}
|
||||
delegate.completeNext(t, nil)
|
||||
if got := outboxBacklog(sink); got != 0 {
|
||||
t.Fatalf("backlog after replay ack = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxReplaysPreexistingRecordAfterRestart(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
first := &outboxAsyncSink{submitErrors: []error{errors.New("nats unavailable")}}
|
||||
writer, err := NewDurableOutboxSink(first, DurableOutboxConfig{Directory: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("create first outbox: %v", err)
|
||||
}
|
||||
if err := writer.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("persist restart record: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close first outbox: %v", err)
|
||||
}
|
||||
delegate := &outboxAsyncSink{}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir, ReplayBatchSize: 10})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
if got := delegate.recordCount(); got != 1 {
|
||||
t.Fatalf("replayed records = %d, want 1", got)
|
||||
}
|
||||
delegate.completeNext(t, nil)
|
||||
if got := outboxBacklog(sink); got != 0 {
|
||||
t.Fatalf("backlog after replay ack = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxDoesNotResubmitInflightRecord(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.ReplayOnce(context.Background()); err != nil {
|
||||
t.Fatalf("ReplayOnce() error = %v", err)
|
||||
}
|
||||
if got := delegate.recordCount(); got != 1 {
|
||||
t.Fatalf("submissions while inflight = %d, want 1", got)
|
||||
}
|
||||
delegate.completeNext(t, nil)
|
||||
}
|
||||
|
||||
func TestDurableOutboxRejectsInvalidRecordBeforePersistence(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{validateErr: errors.New("subject not configured")}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
|
||||
err = sink.PublishRaw(context.Background(), durableTestEnvelope())
|
||||
if err == nil || !strings.Contains(err.Error(), "subject not configured") {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if got := outboxBacklog(sink); got != 0 {
|
||||
t.Fatalf("invalid record backlog = %d, want 0", got)
|
||||
}
|
||||
if got := delegate.recordCount(); got != 0 {
|
||||
t.Fatalf("invalid record submissions = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxCloseWaitsForPendingAck(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
|
||||
Directory: dir,
|
||||
CloseTimeout: time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- sink.Close() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("Close() returned before ack: %v", err)
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
delegate.completeNext(t, nil)
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close() did not return after ack")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableOutboxCloseTimeoutRetainsRecord(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
delegate := &outboxAsyncSink{}
|
||||
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
|
||||
Directory: dir,
|
||||
CloseTimeout: 10 * time.Millisecond,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
|
||||
err = sink.Close()
|
||||
if err == nil || !strings.Contains(err.Error(), "close timed out") {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
if got := outboxBacklog(sink); got != 1 {
|
||||
t.Fatalf("backlog after close timeout = %d, want 1", got)
|
||||
}
|
||||
delegate.completeNext(t, errors.New("connection closed"))
|
||||
}
|
||||
|
||||
func BenchmarkDurableOutboxPublishRaw(b *testing.B) {
|
||||
for _, syncWrites := range []bool{false, true} {
|
||||
name := "fsync_off"
|
||||
if syncWrites {
|
||||
name = "fsync_on"
|
||||
}
|
||||
b.Run(name, func(b *testing.B) {
|
||||
sink, err := NewDurableOutboxSink(autoAckOutboxSink{}, DurableOutboxConfig{
|
||||
Directory: b.TempDir(),
|
||||
SyncWrites: syncWrites,
|
||||
})
|
||||
if err != nil {
|
||||
b.Fatalf("NewDurableOutboxSink() error = %v", err)
|
||||
}
|
||||
var sequence atomic.Uint32
|
||||
b.ResetTimer()
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
for pb.Next() {
|
||||
env := durableTestEnvelope()
|
||||
env.EventID = ""
|
||||
env.Sequence = uint16(sequence.Add(1))
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
b.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
b.StopTimer()
|
||||
if err := sink.Close(); err != nil {
|
||||
b.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func outboxBacklog(sink *DurableOutboxSink) int {
|
||||
count, _ := sink.store.Stats()
|
||||
return count
|
||||
}
|
||||
|
||||
type outboxAsyncSink struct {
|
||||
mu sync.Mutex
|
||||
validateErr error
|
||||
submitErrors []error
|
||||
records []durableRecord
|
||||
callbacks []func(error)
|
||||
onSubmit func(durableRecord)
|
||||
closeCalls int
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) ValidateRecord(record durableRecord) error {
|
||||
if s.validateErr != nil {
|
||||
return s.validateErr
|
||||
}
|
||||
switch record.Kind {
|
||||
case "raw", "unified", "fields":
|
||||
return nil
|
||||
default:
|
||||
return errUnknownRecordKind(record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) PublishRecordAsync(record durableRecord, complete func(error)) error {
|
||||
if s.onSubmit != nil {
|
||||
s.onSubmit(record)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(s.submitErrors) > 0 {
|
||||
err := s.submitErrors[0]
|
||||
s.submitErrors = s.submitErrors[1:]
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
s.records = append(s.records, record)
|
||||
s.callbacks = append(s.callbacks, complete)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) Close() error {
|
||||
s.mu.Lock()
|
||||
s.closeCalls++
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) completeNext(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
s.mu.Lock()
|
||||
if len(s.callbacks) == 0 {
|
||||
s.mu.Unlock()
|
||||
t.Fatal("no pending async callback")
|
||||
}
|
||||
callback := s.callbacks[0]
|
||||
s.callbacks = s.callbacks[1:]
|
||||
s.mu.Unlock()
|
||||
callback(err)
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) recordCount() int {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return len(s.records)
|
||||
}
|
||||
|
||||
func (s *outboxAsyncSink) recordAt(index int) durableRecord {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.records[index]
|
||||
}
|
||||
|
||||
type autoAckOutboxSink struct{}
|
||||
|
||||
func (autoAckOutboxSink) ValidateRecord(record durableRecord) error {
|
||||
switch record.Kind {
|
||||
case "raw", "unified", "fields":
|
||||
return nil
|
||||
default:
|
||||
return errUnknownRecordKind(record.Kind)
|
||||
}
|
||||
}
|
||||
|
||||
func (autoAckOutboxSink) PublishRecordAsync(_ durableRecord, complete func(error)) error {
|
||||
complete(nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (autoAckOutboxSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (autoAckOutboxSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (autoAckOutboxSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (autoAckOutboxSink) Close() error { return nil }
|
||||
Reference in New Issue
Block a user