shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
go/vehicle-gateway/internal/eventbus/async_sink.go
View File

@@ -13,6 +13,7 @@ import (
type AsyncConfig struct {
QueueSize int
Workers int
EnqueueTimeout time.Duration
OperationTimeout time.Duration
OnError func(error)
Metrics *metrics.Registry
@@ -20,12 +21,14 @@ type AsyncConfig struct {
}
type AsyncSink struct {
delegate Sink
jobs chan asyncJob
timeout time.Duration
onError func(error)
metrics *metrics.Registry
name string
delegate Sink
jobs chan asyncJob
enqueueTimeout time.Duration
timeout time.Duration
onError func(error)
metrics *metrics.Registry
name string
queueWait *metrics.RecentLatencyByKey
closeOnce sync.Once
closed chan struct{}
@@ -34,11 +37,13 @@ type AsyncSink struct {
}
type asyncJob struct {
kind string
env envelope.FrameEnvelope
kind string
env envelope.FrameEnvelope
enqueuedAt time.Time
}
var ErrAsyncSinkClosed = errors.New("async sink is closed")
var ErrAsyncSinkEnqueueTimeout = errors.New("async sink enqueue timeout")
var asyncSinkPublishDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
@@ -52,6 +57,12 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink {
if cfg.Workers <= 0 {
cfg.Workers = 1
}
if cfg.EnqueueTimeout == 0 {
cfg.EnqueueTimeout = time.Second
}
if cfg.EnqueueTimeout < 0 {
cfg.EnqueueTimeout = 0
}
if cfg.OperationTimeout <= 0 {
cfg.OperationTimeout = 30 * time.Second
}
@@ -59,19 +70,23 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink {
cfg.Name = "async"
}
s := &AsyncSink{
delegate: delegate,
jobs: make(chan asyncJob, cfg.QueueSize),
timeout: cfg.OperationTimeout,
onError: cfg.OnError,
metrics: cfg.Metrics,
name: cfg.Name,
closed: make(chan struct{}),
done: make(chan struct{}),
delegate: delegate,
jobs: make(chan asyncJob, cfg.QueueSize),
enqueueTimeout: cfg.EnqueueTimeout,
timeout: cfg.OperationTimeout,
onError: cfg.OnError,
metrics: cfg.Metrics,
name: cfg.Name,
queueWait: metrics.NewRecentLatencyByKey(512),
closed: make(chan struct{}),
done: make(chan struct{}),
}
s.wg.Add(cfg.Workers)
for i := 0; i < cfg.Workers; i++ {
go s.worker()
}
s.recordQueueCapacity()
s.recordWorkers("default", cfg.Workers)
go func() {
s.wg.Wait()
close(s.done)
@@ -94,7 +109,6 @@ func (s *AsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelop
func (s *AsyncSink) Close() error {
s.closeOnce.Do(func() {
close(s.closed)
close(s.jobs)
})
<-s.done
return s.delegate.Close()
@@ -107,6 +121,14 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error {
return ErrAsyncSinkClosed
default:
}
var timeoutC <-chan time.Time
var timer *time.Timer
if s.enqueueTimeout > 0 {
timer = time.NewTimer(s.enqueueTimeout)
timeoutC = timer.C
defer timer.Stop()
}
job.enqueuedAt = time.Now()
select {
case s.jobs <- job:
s.recordEnqueue(job.kind, "queued")
@@ -119,37 +141,58 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error {
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth()
return ctx.Err()
case <-timeoutC:
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth()
return ErrAsyncSinkEnqueueTimeout
}
}
func (s *AsyncSink) worker() {
defer s.wg.Done()
for job := range s.jobs {
s.recordQueueDepth()
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
started := time.Now()
var err error
switch job.kind {
case "raw":
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
for {
select {
case job := <-s.jobs:
s.publishJob(job)
case <-s.closed:
for {
select {
case job := <-s.jobs:
s.publishJob(job)
default:
return
}
}
}
cancel()
status := "ok"
if err != nil {
status = "error"
}
s.recordPublish(job.kind, status, time.Since(started))
if err != nil && s.onError != nil {
s.onError(err)
}
s.recordQueueDepth()
}
}
func (s *AsyncSink) publishJob(job asyncJob) {
s.recordQueueDepth()
s.recordQueueWait("default", job)
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
started := time.Now()
var err error
switch job.kind {
case "raw":
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
}
cancel()
status := "ok"
if err != nil {
status = "error"
}
s.recordPublish(job.kind, status, time.Since(started))
if err != nil && s.onError != nil {
s.onError(err)
}
s.recordQueueDepth()
}
func (s *AsyncSink) recordEnqueue(kind string, status string) {
if s.metrics == nil {
return
@@ -184,3 +227,38 @@ func (s *AsyncSink) recordQueueDepth() {
"sink": s.name,
}, float64(len(s.jobs)))
}
func (s *AsyncSink) recordQueueCapacity() {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_queue_capacity", metrics.Labels{
"sink": s.name,
}, float64(cap(s.jobs)))
}
func (s *AsyncSink) recordQueueWait(queueName string, job asyncJob) {
if s.metrics == nil || job.enqueuedAt.IsZero() {
return
}
elapsedMS := float64(time.Since(job.enqueuedAt)) / float64(time.Millisecond)
labels := metrics.Labels{
"sink": s.name,
"queue": queueName,
"kind": job.kind,
}
s.metrics.ObserveHistogram("vehicle_async_sink_queue_wait_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS)
p99, samples := s.queueWait.Observe(queueName+"\x00"+job.kind, elapsedMS)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_p99_ms", labels, p99)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_samples", labels, float64(samples))
}
func (s *AsyncSink) recordWorkers(queueName string, workers int) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_workers", metrics.Labels{
"sink": s.name,
"queue": queueName,
}, float64(workers))
}

145
go/vehicle-gateway/internal/eventbus/async_sink_test.go
View File

@@ -2,6 +2,7 @@ package eventbus
import (
"context"
"errors"
"strings"
"testing"
"time"
@@ -80,12 +81,17 @@ func TestAsyncSinkRecordsQueueMetrics(t *testing.T) {
for _, want := range []string{
`vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`,
`vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`,
`vehicle_async_sink_queue_capacity{sink="nats"} 2`,
`vehicle_async_sink_queue_depth{sink="nats"}`,
`vehicle_async_sink_workers{queue="default",sink="nats"} 1`,
`vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_bucket{le="+Inf",kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_count{kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_sum{kind="raw",sink="nats",status="ok"}`,
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="default",sink="nats"} 1`,
`vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="default",sink="nats"}`,
`vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="default",sink="nats"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("async sink metric missing %s:\n%s", want, text)
@@ -135,6 +141,145 @@ func TestAsyncSinkRecordsEnqueueTimeoutWhenQueueIsFull(t *testing.T) {
delegate.release()
}
func TestAsyncSinkEnqueueTimeoutDoesNotRequireCallerDeadline(t *testing.T) {
registry := metrics.NewRegistry()
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: 10 * time.Millisecond,
OperationTimeout: time.Second,
Metrics: registry,
Name: "nats",
})
defer sink.Close()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); !errors.Is(err, ErrAsyncSinkEnqueueTimeout) {
t.Fatalf("third PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err)
}
text := registry.Render()
for _, want := range []string{
`vehicle_async_sink_enqueue_total{kind="unified",sink="nats",status="timeout"} 1`,
`vehicle_async_sink_queue_depth{sink="nats"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("async sink enqueue timeout metric missing %s:\n%s", want, text)
}
}
delegate.release()
}
func TestAsyncSinkCloseDoesNotPanicWithConcurrentBlockedEnqueue(t *testing.T) {
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: 20 * time.Millisecond,
OperationTimeout: time.Second,
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
publishErr := make(chan error, 1)
go func() {
defer func() {
if recovered := recover(); recovered != nil {
publishErr <- errors.New("publish panicked")
}
}()
publishErr <- sink.PublishUnified(context.Background(), env)
}()
closeErr := make(chan error, 1)
go func() {
closeErr <- sink.Close()
}()
if err := <-publishErr; !errors.Is(err, ErrAsyncSinkEnqueueTimeout) && !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("concurrent PublishUnified() error = %v, want timeout or closed", err)
}
delegate.release()
if err := <-closeErr; err != nil {
t.Fatalf("Close() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), env); !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("PublishRaw() after Close error = %v, want ErrAsyncSinkClosed", err)
}
}
func TestAsyncSinkCloseUnblocksPublishWhenEnqueueTimeoutIsDisabled(t *testing.T) {
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: -1,
OperationTimeout: time.Second,
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
publishErr := make(chan error, 1)
go func() {
publishErr <- sink.PublishUnified(context.Background(), env)
}()
closeErr := make(chan error, 1)
go func() {
closeErr <- sink.Close()
}()
select {
case err := <-publishErr:
if !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err)
}
case <-time.After(time.Second):
t.Fatal("blocked PublishUnified() was not released by Close")
}
delegate.release()
select {
case err := <-closeErr:
if err != nil {
t.Fatalf("Close() error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close() did not finish after delegate release")
}
}
type blockingSink struct {
rawStarted chan struct{}
releaseRaw chan struct{}

348
go/vehicle-gateway/internal/eventbus/durable_outbox_sink.go Normal file
View File

@@ -0,0 +1,348 @@
package eventbus
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
type DurableOutboxConfig struct {
Directory string
ReplayBatchSize int
SyncWrites bool
CloseTimeout time.Duration
WALSegmentBytes int64
WALSegmentAge time.Duration
WALAppendQueue int
WALCommitBatch int
WALCommitWait time.Duration
Metrics *metrics.Registry
Name string
OnError func(error)
}
type asyncRecordPublishingSink interface {
Sink
ValidateRecord(durableRecord) error
PublishRecordAsync(durableRecord, func(error)) error
}
// DurableOutboxSink accepts a record only after its WAL commit is durable.
// Publishing is asynchronous; the WAL record remains replayable until the
// broker returns PubAck. Stable event IDs make crash-window replays idempotent.
type DurableOutboxSink struct {
delegate asyncRecordPublishingSink
store *durableOutboxWAL
replayBatchSize int
closeTimeout time.Duration
metrics *metrics.Registry
name string
onError func(error)
acceptMu sync.RWMutex
mu sync.Mutex
closed bool
inflight map[outboxWALRecordRef]struct{}
pending sync.WaitGroup
closeOne sync.Once
closeErr error
}
var ErrDurableOutboxClosed = errors.New("durable outbox is closed")
func NewDurableOutboxSink(delegate Sink, cfg DurableOutboxConfig) (*DurableOutboxSink, error) {
publisher, ok := delegate.(asyncRecordPublishingSink)
if !ok || publisher == nil {
return nil, errors.New("durable outbox delegate must support async record publishing")
}
dir := strings.TrimSpace(cfg.Directory)
if dir == "" {
return nil, errors.New("durable outbox directory is required")
}
if cfg.ReplayBatchSize <= 0 {
cfg.ReplayBatchSize = 1000
}
if cfg.CloseTimeout <= 0 {
cfg.CloseTimeout = 5 * time.Second
}
store, err := newDurableOutboxWAL(durableOutboxWALConfig{
Directory: dir,
SyncWrites: cfg.SyncWrites,
SegmentBytes: cfg.WALSegmentBytes,
SegmentAge: cfg.WALSegmentAge,
AppendQueue: cfg.WALAppendQueue,
CommitBatch: cfg.WALCommitBatch,
CommitInterval: cfg.WALCommitWait,
})
if err != nil {
return nil, err
}
s := &DurableOutboxSink{
delegate: publisher,
store: store,
replayBatchSize: cfg.ReplayBatchSize,
closeTimeout: cfg.CloseTimeout,
metrics: cfg.Metrics,
name: durableMetricName(cfg.Name),
onError: cfg.OnError,
inflight: map[outboxWALRecordRef]struct{}{},
}
s.recordBacklog()
return s, nil
}
func (s *DurableOutboxSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
return s.persistAndSubmit(ctx, "raw", env)
}
func (s *DurableOutboxSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.persistAndSubmit(ctx, "unified", env)
}
func (s *DurableOutboxSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
return s.persistAndSubmit(ctx, "fields", env)
}
func (s *DurableOutboxSink) Close() error {
s.closeOne.Do(func() {
s.acceptMu.Lock()
s.mu.Lock()
s.closed = true
s.mu.Unlock()
s.acceptMu.Unlock()
walErr := s.store.Close()
done := make(chan struct{})
go func() {
s.pending.Wait()
close(done)
}()
var waitErr error
select {
case <-done:
case <-time.After(s.closeTimeout):
waitErr = fmt.Errorf("durable outbox close timed out after %s", s.closeTimeout)
s.recordPublish("all", "close_timeout")
}
s.closeErr = errors.Join(walErr, waitErr, s.delegate.Close())
})
return s.closeErr
}
// ReplayOnce claims at most one configured batch. A bounded claim prevents a
// large recovered backlog from creating an unbounded number of PubAck futures.
func (s *DurableOutboxSink) ReplayOnce(ctx context.Context) error {
s.acceptMu.RLock()
defer s.acceptMu.RUnlock()
if s.isClosed() {
return ErrDurableOutboxClosed
}
return s.replay(ctx, s.replayBatchSize)
}
func (s *DurableOutboxSink) ReplayLoop(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = time.Second
}
if err := s.ReplayOnce(ctx); err != nil && !errors.Is(err, ErrDurableOutboxClosed) {
s.reportError(err)
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
if err := s.ReplayOnce(ctx); err != nil && !errors.Is(err, ErrDurableOutboxClosed) {
s.reportError(err)
}
}
}
}
func (s *DurableOutboxSink) persistAndSubmit(ctx context.Context, kind string, env envelope.FrameEnvelope) error {
s.acceptMu.RLock()
defer s.acceptMu.RUnlock()
if s.isClosed() {
return ErrDurableOutboxClosed
}
record := durableRecord{Kind: kind, Envelope: normalizeDurableEnvelope(env)}
if err := s.delegate.ValidateRecord(record); err != nil {
s.recordPublish(kind, "validation_error")
return err
}
stored, err := s.store.Append(ctx, record)
if err != nil {
s.recordSpool(kind, "error")
return err
}
s.recordSpool(kind, "ok")
s.recordBacklog()
if err := s.submit(stored); err != nil {
// The durable commit is the device-facing acceptance boundary. Broker
// submission errors are surfaced operationally and recovered by replay.
s.reportError(err)
}
return nil
}
func normalizeDurableEnvelope(env envelope.FrameEnvelope) envelope.FrameEnvelope {
if env.EventID == "" {
env.EventID = env.StableEventID()
}
if env.ParseStatus == "" {
env.ParseStatus = envelope.ParseOK
}
return env
}
func (s *DurableOutboxSink) submit(stored storedOutboxRecord) error {
s.mu.Lock()
if s.closed {
s.mu.Unlock()
s.store.Release(stored.Ref)
return ErrDurableOutboxClosed
}
if _, exists := s.inflight[stored.Ref]; exists {
s.mu.Unlock()
return nil
}
s.inflight[stored.Ref] = struct{}{}
s.pending.Add(1)
inflight := len(s.inflight)
s.mu.Unlock()
s.recordInflight(inflight)
err := s.delegate.PublishRecordAsync(stored.Record, func(publishErr error) {
s.complete(stored, publishErr)
})
if err == nil {
s.recordPublish(stored.Record.Kind, "submitted")
return nil
}
s.mu.Lock()
delete(s.inflight, stored.Ref)
inflight = len(s.inflight)
s.mu.Unlock()
s.pending.Done()
s.store.Release(stored.Ref)
s.recordInflight(inflight)
s.recordPublish(stored.Record.Kind, "submit_error")
return fmt.Errorf("submit durable outbox record: %w", err)
}
func (s *DurableOutboxSink) complete(stored storedOutboxRecord, publishErr error) {
defer s.pending.Done()
status := "acked"
if publishErr != nil {
status = "ack_error"
s.store.Release(stored.Ref)
} else if err := s.store.Ack(stored.Ref); err != nil {
publishErr = err
status = "remove_error"
}
s.mu.Lock()
delete(s.inflight, stored.Ref)
inflight := len(s.inflight)
s.mu.Unlock()
s.recordInflight(inflight)
s.recordBacklog()
s.recordPublish(stored.Record.Kind, status)
if publishErr != nil {
s.reportError(publishErr)
}
}
func (s *DurableOutboxSink) replay(ctx context.Context, limit int) error {
if ctx != nil && ctx.Err() != nil {
return ctx.Err()
}
records, err := s.store.ClaimPending(limit)
if err != nil {
return fmt.Errorf("claim durable outbox records: %w", err)
}
for index, stored := range records {
if ctx != nil && ctx.Err() != nil {
for _, remaining := range records[index:] {
s.store.Release(remaining.Ref)
}
return ctx.Err()
}
if err := s.delegate.ValidateRecord(stored.Record); err != nil {
s.store.Release(stored.Ref)
for _, remaining := range records[index+1:] {
s.store.Release(remaining.Ref)
}
s.recordPublish(stored.Record.Kind, "validation_error")
return fmt.Errorf("validate durable outbox replay record: %w", err)
}
if err := s.submit(stored); err != nil && !errors.Is(err, ErrDurableOutboxClosed) {
s.reportError(err)
}
}
return nil
}
func (s *DurableOutboxSink) isClosed() bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.closed
}
func (s *DurableOutboxSink) reportError(err error) {
if err != nil && s.onError != nil {
s.onError(err)
}
}
func (s *DurableOutboxSink) recordSpool(kind string, status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{
"name": s.name, "kind": kind, "status": status,
})
}
func (s *DurableOutboxSink) recordPublish(kind string, status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_outbox_publish_total", metrics.Labels{
"name": s.name, "kind": kind, "status": status,
})
}
func (s *DurableOutboxSink) recordInflight(value int) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_durable_outbox_inflight", metrics.Labels{"name": s.name}, float64(value))
}
func (s *DurableOutboxSink) recordBacklog() {
if s.metrics == nil {
return
}
count, oldest := s.store.Stats()
labels := metrics.Labels{"name": s.name}
// Keep the legacy gauge during migration because capacity gates already
// consume it; its value now represents durable WAL records, not files.
s.metrics.SetGauge("vehicle_durable_spool_backlog_files", labels, float64(count))
s.metrics.SetGauge("vehicle_durable_outbox_backlog_records", labels, float64(count))
ageSeconds := 0.0
if count > 0 && !oldest.IsZero() {
ageSeconds = time.Since(oldest).Seconds()
if ageSeconds < 0 {
ageSeconds = 0
}
}
s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", labels, ageSeconds)
}

380
go/vehicle-gateway/internal/eventbus/durable_outbox_sink_test.go Normal file
View File

@@ -0,0 +1,380 @@
package eventbus
import (
"context"
"errors"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
func TestDurableOutboxPersistsBeforeAsyncSubmitAndDeletesAfterAck(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{}
var sink *DurableOutboxSink
delegate.onSubmit = func(record durableRecord) {
if got := outboxBacklog(sink); got != 1 {
t.Fatalf("durable backlog visible at submit = %d, want 1", got)
}
if record.Envelope.EventID == "" {
t.Fatal("submitted record must have a stable event id")
}
}
var err error
sink, err = NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if got := outboxBacklog(sink); got != 1 {
t.Fatalf("backlog before ack = %d, want 1", got)
}
delegate.completeNext(t, nil)
if got := outboxBacklog(sink); got != 0 {
t.Fatalf("backlog after ack = %d, want 0", got)
}
}
func TestDurableOutboxImmediateSubmitErrorKeepsAcceptedRecord(t *testing.T) {
dir := t.TempDir()
submitErr := errors.New("nats pending limit")
delegate := &outboxAsyncSink{submitErrors: []error{submitErr}}
var reported error
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
Directory: dir,
OnError: func(err error) {
reported = err
},
})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("accepted durable record should hide submit error, got %v", err)
}
if got := outboxBacklog(sink); got != 1 {
t.Fatalf("backlog after submit error = %d, want 1", got)
}
if reported == nil || !strings.Contains(reported.Error(), submitErr.Error()) {
t.Fatalf("reported error = %v", reported)
}
}
func TestDurableOutboxAckErrorIsRetriedAndStableRecordIsRemoved(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
delegate.completeNext(t, errors.New("ack timeout"))
if got := outboxBacklog(sink); got != 1 {
t.Fatalf("backlog after ack error = %d, want 1", got)
}
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
if got := delegate.recordCount(); got != 2 {
t.Fatalf("async submissions = %d, want 2", got)
}
first, second := delegate.recordAt(0), delegate.recordAt(1)
if first.Envelope.EventID == "" || first.Envelope.EventID != second.Envelope.EventID {
t.Fatalf("replay event ids = %q and %q", first.Envelope.EventID, second.Envelope.EventID)
}
delegate.completeNext(t, nil)
if got := outboxBacklog(sink); got != 0 {
t.Fatalf("backlog after replay ack = %d, want 0", got)
}
}
func TestDurableOutboxReplaysPreexistingRecordAfterRestart(t *testing.T) {
dir := t.TempDir()
first := &outboxAsyncSink{submitErrors: []error{errors.New("nats unavailable")}}
writer, err := NewDurableOutboxSink(first, DurableOutboxConfig{Directory: dir})
if err != nil {
t.Fatalf("create first outbox: %v", err)
}
if err := writer.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("persist restart record: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close first outbox: %v", err)
}
delegate := &outboxAsyncSink{}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir, ReplayBatchSize: 10})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
if got := delegate.recordCount(); got != 1 {
t.Fatalf("replayed records = %d, want 1", got)
}
delegate.completeNext(t, nil)
if got := outboxBacklog(sink); got != 0 {
t.Fatalf("backlog after replay ack = %d, want 0", got)
}
}
func TestDurableOutboxDoesNotResubmitInflightRecord(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
if got := delegate.recordCount(); got != 1 {
t.Fatalf("submissions while inflight = %d, want 1", got)
}
delegate.completeNext(t, nil)
}
func TestDurableOutboxRejectsInvalidRecordBeforePersistence(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{validateErr: errors.New("subject not configured")}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{Directory: dir})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
err = sink.PublishRaw(context.Background(), durableTestEnvelope())
if err == nil || !strings.Contains(err.Error(), "subject not configured") {
t.Fatalf("PublishRaw() error = %v", err)
}
if got := outboxBacklog(sink); got != 0 {
t.Fatalf("invalid record backlog = %d, want 0", got)
}
if got := delegate.recordCount(); got != 0 {
t.Fatalf("invalid record submissions = %d, want 0", got)
}
}
func TestDurableOutboxCloseWaitsForPendingAck(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
Directory: dir,
CloseTimeout: time.Second,
})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
done := make(chan error, 1)
go func() { done <- sink.Close() }()
select {
case err := <-done:
t.Fatalf("Close() returned before ack: %v", err)
case <-time.After(20 * time.Millisecond):
}
delegate.completeNext(t, nil)
select {
case err := <-done:
if err != nil {
t.Fatalf("Close() error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close() did not return after ack")
}
}
func TestDurableOutboxCloseTimeoutRetainsRecord(t *testing.T) {
dir := t.TempDir()
delegate := &outboxAsyncSink{}
sink, err := NewDurableOutboxSink(delegate, DurableOutboxConfig{
Directory: dir,
CloseTimeout: 10 * time.Millisecond,
})
if err != nil {
t.Fatalf("NewDurableOutboxSink() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), durableTestEnvelope()); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
err = sink.Close()
if err == nil || !strings.Contains(err.Error(), "close timed out") {
t.Fatalf("Close() error = %v", err)
}
if got := outboxBacklog(sink); got != 1 {
t.Fatalf("backlog after close timeout = %d, want 1", got)
}
delegate.completeNext(t, errors.New("connection closed"))
}
func BenchmarkDurableOutboxPublishRaw(b *testing.B) {
for _, syncWrites := range []bool{false, true} {
name := "fsync_off"
if syncWrites {
name = "fsync_on"
}
b.Run(name, func(b *testing.B) {
sink, err := NewDurableOutboxSink(autoAckOutboxSink{}, DurableOutboxConfig{
Directory: b.TempDir(),
SyncWrites: syncWrites,
})
if err != nil {
b.Fatalf("NewDurableOutboxSink() error = %v", err)
}
var sequence atomic.Uint32
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
env := durableTestEnvelope()
env.EventID = ""
env.Sequence = uint16(sequence.Add(1))
if err := sink.PublishRaw(context.Background(), env); err != nil {
b.Fatalf("PublishRaw() error = %v", err)
}
}
})
b.StopTimer()
if err := sink.Close(); err != nil {
b.Fatalf("Close() error = %v", err)
}
})
}
}
func outboxBacklog(sink *DurableOutboxSink) int {
count, _ := sink.store.Stats()
return count
}
type outboxAsyncSink struct {
mu sync.Mutex
validateErr error
submitErrors []error
records []durableRecord
callbacks []func(error)
onSubmit func(durableRecord)
closeCalls int
}
func (s *outboxAsyncSink) ValidateRecord(record durableRecord) error {
if s.validateErr != nil {
return s.validateErr
}
switch record.Kind {
case "raw", "unified", "fields":
return nil
default:
return errUnknownRecordKind(record.Kind)
}
}
func (s *outboxAsyncSink) PublishRecordAsync(record durableRecord, complete func(error)) error {
if s.onSubmit != nil {
s.onSubmit(record)
}
s.mu.Lock()
defer s.mu.Unlock()
if len(s.submitErrors) > 0 {
err := s.submitErrors[0]
s.submitErrors = s.submitErrors[1:]
if err != nil {
return err
}
}
s.records = append(s.records, record)
s.callbacks = append(s.callbacks, complete)
return nil
}
func (s *outboxAsyncSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *outboxAsyncSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *outboxAsyncSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *outboxAsyncSink) Close() error {
s.mu.Lock()
s.closeCalls++
s.mu.Unlock()
return nil
}
func (s *outboxAsyncSink) completeNext(t *testing.T, err error) {
t.Helper()
s.mu.Lock()
if len(s.callbacks) == 0 {
s.mu.Unlock()
t.Fatal("no pending async callback")
}
callback := s.callbacks[0]
s.callbacks = s.callbacks[1:]
s.mu.Unlock()
callback(err)
}
func (s *outboxAsyncSink) recordCount() int {
s.mu.Lock()
defer s.mu.Unlock()
return len(s.records)
}
func (s *outboxAsyncSink) recordAt(index int) durableRecord {
s.mu.Lock()
defer s.mu.Unlock()
return s.records[index]
}
type autoAckOutboxSink struct{}
func (autoAckOutboxSink) ValidateRecord(record durableRecord) error {
switch record.Kind {
case "raw", "unified", "fields":
return nil
default:
return errUnknownRecordKind(record.Kind)
}
}
func (autoAckOutboxSink) PublishRecordAsync(_ durableRecord, complete func(error)) error {
complete(nil)
return nil
}
func (autoAckOutboxSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (autoAckOutboxSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (autoAckOutboxSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (autoAckOutboxSink) Close() error { return nil }

804
go/vehicle-gateway/internal/eventbus/durable_outbox_wal.go Normal file
View File

@@ -0,0 +1,804 @@
package eventbus
import (
"context"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"hash/crc32"
"io"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
)
const (
outboxWALMagic = uint32(0x4c4e5731) // LNW1
outboxWALHeaderSize = 12
outboxWALMaxRecordBytes = 16 << 20
defaultWALSegmentBytes = 16 << 20
defaultWALAppendQueue = 100_000
defaultWALCommitBatch = 256
defaultWALCommitInterval = time.Millisecond
defaultWALSegmentAge = 5 * time.Second
)
var ErrDurableOutboxWALClosed = errors.New("durable outbox wal is closed")
type durableOutboxWALConfig struct {
Directory string
SyncWrites bool
SegmentBytes int64
SegmentAge time.Duration
AppendQueue int
CommitBatch int
CommitInterval time.Duration
}
type outboxWALRecordRef struct {
segmentID uint64
index int
}
type storedOutboxRecord struct {
Ref outboxWALRecordRef
Record durableRecord
}
type outboxWALRecordStatus uint8
const (
outboxWALPending outboxWALRecordStatus = iota
outboxWALClaimed
outboxWALAcknowledged
)
type outboxWALRecordState struct {
payloadOffset int64
payloadLength uint32
status outboxWALRecordStatus
}
type outboxWALSegment struct {
id uint64
path string
createdAt time.Time
size int64
closed bool
deleting bool
acked int
records []*outboxWALRecordState
}
type outboxWALAppendRequest struct {
record durableRecord
payload []byte
frame []byte
result chan outboxWALAppendResult
}
type outboxWALAppendResult struct {
stored storedOutboxRecord
err error
}
type durableOutboxWAL struct {
dir string
syncWrites bool
segmentBytes int64
segmentAge time.Duration
commitBatch int
commitInterval time.Duration
mu sync.Mutex
segments []*outboxWALSegment
segmentByID map[uint64]*outboxWALSegment
current *outboxWALSegment
currentFile *os.File
backlog int
fatalErr error
appendMu sync.RWMutex
closed bool
queue chan outboxWALAppendRequest
writerWG sync.WaitGroup
closeOne sync.Once
}
func newDurableOutboxWAL(cfg durableOutboxWALConfig) (*durableOutboxWAL, error) {
dir := strings.TrimSpace(cfg.Directory)
if dir == "" {
return nil, errors.New("durable outbox wal directory is required")
}
if cfg.SegmentBytes <= 0 {
cfg.SegmentBytes = defaultWALSegmentBytes
}
if cfg.SegmentAge <= 0 {
cfg.SegmentAge = defaultWALSegmentAge
}
if cfg.AppendQueue <= 0 {
cfg.AppendQueue = defaultWALAppendQueue
}
if cfg.CommitBatch <= 0 {
cfg.CommitBatch = defaultWALCommitBatch
}
if cfg.CommitInterval <= 0 {
cfg.CommitInterval = defaultWALCommitInterval
}
if err := os.MkdirAll(dir, 0o750); err != nil {
return nil, fmt.Errorf("create durable outbox wal directory: %w", err)
}
w := &durableOutboxWAL{
dir: dir,
syncWrites: cfg.SyncWrites,
segmentBytes: cfg.SegmentBytes,
segmentAge: cfg.SegmentAge,
commitBatch: cfg.CommitBatch,
commitInterval: cfg.CommitInterval,
segmentByID: map[uint64]*outboxWALSegment{},
queue: make(chan outboxWALAppendRequest, cfg.AppendQueue),
}
if err := w.loadSegments(); err != nil {
return nil, err
}
if err := w.createCurrentSegment(); err != nil {
return nil, err
}
w.writerWG.Add(1)
go w.appendLoop()
return w, nil
}
func (w *durableOutboxWAL) Append(ctx context.Context, record durableRecord) (storedOutboxRecord, error) {
payload, err := json.Marshal(record)
if err != nil {
return storedOutboxRecord{}, fmt.Errorf("marshal durable outbox wal record: %w", err)
}
if len(payload) > outboxWALMaxRecordBytes {
return storedOutboxRecord{}, fmt.Errorf("durable outbox wal record is %d bytes, max %d", len(payload), outboxWALMaxRecordBytes)
}
request := outboxWALAppendRequest{
record: record,
payload: payload,
frame: encodeOutboxWALFrame(payload),
result: make(chan outboxWALAppendResult, 1),
}
if ctx == nil {
ctx = context.Background()
}
w.appendMu.RLock()
if w.closed {
w.appendMu.RUnlock()
return storedOutboxRecord{}, ErrDurableOutboxWALClosed
}
select {
case w.queue <- request:
w.appendMu.RUnlock()
case <-ctx.Done():
w.appendMu.RUnlock()
return storedOutboxRecord{}, ctx.Err()
}
// Once admitted to the WAL queue, wait for the durability result even if
// the caller context is cancelled. Otherwise a committed record could be
// left claimed with no publisher responsible for it.
result := <-request.result
return result.stored, result.err
}
func (w *durableOutboxWAL) ClaimPending(limit int) ([]storedOutboxRecord, error) {
if limit <= 0 {
limit = defaultWALCommitBatch
}
w.mu.Lock()
refs := make([]outboxWALRecordRef, 0, limit)
for _, segment := range w.segments {
for index, state := range segment.records {
if state.status != outboxWALPending {
continue
}
state.status = outboxWALClaimed
refs = append(refs, outboxWALRecordRef{segmentID: segment.id, index: index})
if len(refs) >= limit {
break
}
}
if len(refs) >= limit {
break
}
}
w.mu.Unlock()
if len(refs) == 0 {
return nil, nil
}
records, err := w.readClaimedRecords(refs)
if err != nil {
for _, ref := range refs {
w.Release(ref)
}
return nil, err
}
return records, nil
}
func (w *durableOutboxWAL) Release(ref outboxWALRecordRef) {
w.mu.Lock()
defer w.mu.Unlock()
state := w.recordStateLocked(ref)
if state != nil && state.status == outboxWALClaimed {
state.status = outboxWALPending
}
}
func (w *durableOutboxWAL) Ack(ref outboxWALRecordRef) error {
w.mu.Lock()
segment := w.segmentByID[ref.segmentID]
if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
w.mu.Unlock()
return nil
}
state := segment.records[ref.index]
if state.status == outboxWALAcknowledged {
w.mu.Unlock()
return nil
}
state.status = outboxWALAcknowledged
segment.acked++
if w.backlog > 0 {
w.backlog--
}
shouldDelete := segment.closed && segment.acked == len(segment.records) && !segment.deleting
if shouldDelete {
segment.deleting = true
}
w.mu.Unlock()
if !shouldDelete {
return nil
}
return w.deleteAcknowledgedSegment(segment)
}
func (w *durableOutboxWAL) Stats() (backlog int, oldest time.Time) {
w.mu.Lock()
defer w.mu.Unlock()
backlog = w.backlog
if backlog == 0 {
return backlog, time.Time{}
}
for _, segment := range w.segments {
if segment.acked < len(segment.records) {
return backlog, segment.createdAt
}
}
return backlog, time.Time{}
}
func (w *durableOutboxWAL) Close() error {
w.closeOne.Do(func() {
w.appendMu.Lock()
w.closed = true
close(w.queue)
w.appendMu.Unlock()
w.writerWG.Wait()
})
w.mu.Lock()
err := w.fatalErr
w.mu.Unlock()
return err
}
func (w *durableOutboxWAL) appendLoop() {
defer w.writerWG.Done()
maintenanceInterval := minDuration(w.segmentAge/2, time.Second)
if maintenanceInterval < 10*time.Millisecond {
maintenanceInterval = 10 * time.Millisecond
}
maintenance := time.NewTicker(maintenanceInterval)
defer maintenance.Stop()
for {
select {
case request, ok := <-w.queue:
if !ok {
w.finishWriter()
return
}
batch := w.collectAppendBatch(request)
w.commitAppendBatch(batch)
case <-maintenance.C:
if err := w.rotateIfAged(); err != nil {
w.setFatal(err)
}
}
}
}
func (w *durableOutboxWAL) collectAppendBatch(first outboxWALAppendRequest) []outboxWALAppendRequest {
batch := make([]outboxWALAppendRequest, 0, w.commitBatch)
batch = append(batch, first)
timer := time.NewTimer(w.commitInterval)
defer timer.Stop()
for len(batch) < w.commitBatch {
select {
case request, ok := <-w.queue:
if !ok {
return batch
}
batch = append(batch, request)
case <-timer.C:
return batch
}
}
return batch
}
func (w *durableOutboxWAL) commitAppendBatch(batch []outboxWALAppendRequest) {
if len(batch) == 0 {
return
}
w.mu.Lock()
fatalErr := w.fatalErr
w.mu.Unlock()
if fatalErr != nil {
w.completeAppendErrors(batch, fatalErr)
return
}
for len(batch) > 0 {
if err := w.rotateBeforeAppend(len(batch[0].frame)); err != nil {
w.setFatal(err)
w.completeAppendErrors(batch, err)
return
}
count := w.batchCountForCurrentSegment(batch)
if count <= 0 {
count = 1
}
group := batch[:count]
if err := w.commitAppendGroup(group); err != nil {
w.setFatal(err)
w.completeAppendErrors(batch, err)
return
}
batch = batch[count:]
}
}
func (w *durableOutboxWAL) commitAppendGroup(group []outboxWALAppendRequest) error {
start := w.current.size
totalBytes := 0
for _, request := range group {
totalBytes += len(request.frame)
}
buffer := make([]byte, 0, totalBytes)
for _, request := range group {
buffer = append(buffer, request.frame...)
}
written, err := w.currentFile.Write(buffer)
if err != nil || written != len(buffer) {
if err == nil {
err = io.ErrShortWrite
}
w.rollbackAppend(start)
return fmt.Errorf("append durable outbox wal: %w", err)
}
if w.syncWrites {
if err := w.currentFile.Sync(); err != nil {
w.rollbackAppend(start)
return fmt.Errorf("sync durable outbox wal: %w", err)
}
}
w.mu.Lock()
segment := w.current
offset := start
results := make([]outboxWALAppendResult, 0, len(group))
for _, request := range group {
state := &outboxWALRecordState{
payloadOffset: offset + outboxWALHeaderSize,
payloadLength: uint32(len(request.payload)),
status: outboxWALClaimed,
}
index := len(segment.records)
segment.records = append(segment.records, state)
w.backlog++
results = append(results, outboxWALAppendResult{stored: storedOutboxRecord{
Ref: outboxWALRecordRef{segmentID: segment.id, index: index},
Record: request.record,
}})
offset += int64(len(request.frame))
}
segment.size += int64(len(buffer))
w.mu.Unlock()
for index, request := range group {
request.result <- results[index]
}
return nil
}
func (w *durableOutboxWAL) rollbackAppend(size int64) {
if w.currentFile == nil {
return
}
_ = w.currentFile.Truncate(size)
if w.syncWrites {
_ = w.currentFile.Sync()
}
}
func (w *durableOutboxWAL) rotateBeforeAppend(frameBytes int) error {
if w.current == nil {
return w.createCurrentSegment()
}
if len(w.current.records) == 0 {
return nil
}
tooLarge := w.current.size+int64(frameBytes) > w.segmentBytes
tooOld := time.Since(w.current.createdAt) >= w.segmentAge
if !tooLarge && !tooOld {
return nil
}
return w.rotateCurrentSegment()
}
func (w *durableOutboxWAL) rotateIfAged() error {
w.mu.Lock()
current := w.current
shouldRotate := current != nil && len(current.records) > 0 && time.Since(current.createdAt) >= w.segmentAge
w.mu.Unlock()
if !shouldRotate {
return nil
}
return w.rotateCurrentSegment()
}
func (w *durableOutboxWAL) rotateCurrentSegment() error {
if w.currentFile == nil || w.current == nil {
return w.createCurrentSegment()
}
if w.syncWrites {
if err := w.currentFile.Sync(); err != nil {
return fmt.Errorf("sync closing durable outbox wal segment: %w", err)
}
}
if err := w.currentFile.Close(); err != nil {
return fmt.Errorf("close durable outbox wal segment: %w", err)
}
w.mu.Lock()
old := w.current
old.closed = true
w.current = nil
w.currentFile = nil
deleteOld := old.acked == len(old.records) && !old.deleting
if deleteOld {
old.deleting = true
}
w.mu.Unlock()
if deleteOld {
if err := w.deleteAcknowledgedSegment(old); err != nil {
return err
}
}
return w.createCurrentSegment()
}
func (w *durableOutboxWAL) createCurrentSegment() error {
w.mu.Lock()
lastID := uint64(0)
if len(w.segments) > 0 {
lastID = w.segments[len(w.segments)-1].id
}
w.mu.Unlock()
id := uint64(time.Now().UnixNano())
if id <= lastID {
id = lastID + 1
}
path := filepath.Join(w.dir, outboxWALFileName(id))
file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_RDWR|os.O_APPEND, 0o640)
if err != nil {
return fmt.Errorf("create durable outbox wal segment: %w", err)
}
if w.syncWrites {
if err := syncDirectory(w.dir); err != nil {
_ = file.Close()
return fmt.Errorf("sync durable outbox wal directory after create: %w", err)
}
}
segment := &outboxWALSegment{id: id, path: path, createdAt: time.Now()}
w.mu.Lock()
w.segments = append(w.segments, segment)
w.segmentByID[id] = segment
w.current = segment
w.currentFile = file
w.mu.Unlock()
return nil
}
func (w *durableOutboxWAL) finishWriter() {
if w.currentFile == nil || w.current == nil {
return
}
var finishErr error
if w.syncWrites {
finishErr = w.currentFile.Sync()
}
if closeErr := w.currentFile.Close(); finishErr == nil {
finishErr = closeErr
}
w.mu.Lock()
current := w.current
current.closed = true
w.current = nil
w.currentFile = nil
deleteCurrent := current.acked == len(current.records) && !current.deleting
if deleteCurrent {
current.deleting = true
}
w.mu.Unlock()
if deleteCurrent {
if err := w.deleteAcknowledgedSegment(current); finishErr == nil {
finishErr = err
}
}
if finishErr != nil {
w.setFatal(finishErr)
}
}
func (w *durableOutboxWAL) batchCountForCurrentSegment(batch []outboxWALAppendRequest) int {
remaining := w.segmentBytes - w.current.size
count := 0
for _, request := range batch {
if count > 0 && int64(len(request.frame)) > remaining {
break
}
remaining -= int64(len(request.frame))
count++
if remaining <= 0 {
break
}
}
return count
}
func (w *durableOutboxWAL) completeAppendErrors(batch []outboxWALAppendRequest, err error) {
for _, request := range batch {
request.result <- outboxWALAppendResult{err: err}
}
}
func (w *durableOutboxWAL) readClaimedRecords(refs []outboxWALRecordRef) ([]storedOutboxRecord, error) {
files := map[uint64]*os.File{}
defer func() {
for _, file := range files {
_ = file.Close()
}
}()
records := make([]storedOutboxRecord, 0, len(refs))
for _, ref := range refs {
w.mu.Lock()
segment := w.segmentByID[ref.segmentID]
state := w.recordStateLocked(ref)
w.mu.Unlock()
if segment == nil || state == nil {
return nil, fmt.Errorf("durable outbox wal record reference not found: segment=%d index=%d", ref.segmentID, ref.index)
}
file := files[segment.id]
if file == nil {
opened, err := os.Open(segment.path)
if err != nil {
return nil, fmt.Errorf("open durable outbox wal segment for replay: %w", err)
}
files[segment.id] = opened
file = opened
}
payload := make([]byte, state.payloadLength)
if _, err := file.ReadAt(payload, state.payloadOffset); err != nil {
return nil, fmt.Errorf("read durable outbox wal record: %w", err)
}
var record durableRecord
if err := json.Unmarshal(payload, &record); err != nil {
return nil, fmt.Errorf("decode durable outbox wal record: %w", err)
}
records = append(records, storedOutboxRecord{Ref: ref, Record: record})
}
return records, nil
}
func (w *durableOutboxWAL) loadSegments() error {
paths, err := filepath.Glob(filepath.Join(w.dir, "outbox-*.wal"))
if err != nil {
return fmt.Errorf("list durable outbox wal segments: %w", err)
}
sort.Strings(paths)
for _, path := range paths {
segment, err := loadOutboxWALSegment(path, w.syncWrites)
if err != nil {
return err
}
if len(segment.records) == 0 {
if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove empty durable outbox wal segment: %w", err)
}
if w.syncWrites {
if err := syncDirectory(w.dir); err != nil {
return fmt.Errorf("sync durable outbox wal directory after removing empty segment: %w", err)
}
}
continue
}
w.segments = append(w.segments, segment)
w.segmentByID[segment.id] = segment
w.backlog += len(segment.records)
}
return nil
}
func loadOutboxWALSegment(path string, syncWrites bool) (*outboxWALSegment, error) {
id, err := parseOutboxWALFileName(filepath.Base(path))
if err != nil {
return nil, err
}
file, err := os.OpenFile(path, os.O_RDWR, 0)
if err != nil {
return nil, fmt.Errorf("open durable outbox wal segment: %w", err)
}
defer file.Close()
info, err := file.Stat()
if err != nil {
return nil, fmt.Errorf("stat durable outbox wal segment: %w", err)
}
segment := &outboxWALSegment{
id: id,
path: path,
createdAt: info.ModTime(),
closed: true,
}
offset := int64(0)
header := make([]byte, outboxWALHeaderSize)
for offset < info.Size() {
n, readErr := file.ReadAt(header, offset)
if readErr != nil {
if readErr == io.EOF && n < outboxWALHeaderSize {
if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
return nil, err
}
break
}
return nil, fmt.Errorf("read durable outbox wal header at %d: %w", offset, readErr)
}
if binary.BigEndian.Uint32(header[0:4]) != outboxWALMagic {
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid magic", path, offset)
}
length := binary.BigEndian.Uint32(header[4:8])
checksum := binary.BigEndian.Uint32(header[8:12])
if length == 0 || length > outboxWALMaxRecordBytes {
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: invalid length %d", path, offset, length)
}
frameEnd := offset + outboxWALHeaderSize + int64(length)
if frameEnd > info.Size() {
if err := truncateOutboxWALTail(file, offset, syncWrites); err != nil {
return nil, err
}
break
}
payload := make([]byte, length)
if _, err := file.ReadAt(payload, offset+outboxWALHeaderSize); err != nil {
return nil, fmt.Errorf("read durable outbox wal payload at %d: %w", offset, err)
}
if crc32.ChecksumIEEE(payload) != checksum {
return nil, fmt.Errorf("durable outbox wal corruption in %s at offset %d: checksum mismatch", path, offset)
}
segment.records = append(segment.records, &outboxWALRecordState{
payloadOffset: offset + outboxWALHeaderSize,
payloadLength: length,
status: outboxWALPending,
})
offset = frameEnd
}
segment.size = offset
return segment, nil
}
func truncateOutboxWALTail(file *os.File, size int64, syncWrites bool) error {
if err := file.Truncate(size); err != nil {
return fmt.Errorf("truncate incomplete durable outbox wal tail: %w", err)
}
if syncWrites {
if err := file.Sync(); err != nil {
return fmt.Errorf("sync truncated durable outbox wal tail: %w", err)
}
}
return nil
}
func (w *durableOutboxWAL) deleteAcknowledgedSegment(segment *outboxWALSegment) error {
err := os.Remove(segment.path)
if os.IsNotExist(err) {
err = nil
}
if err == nil && w.syncWrites {
err = syncDirectory(w.dir)
}
w.mu.Lock()
defer w.mu.Unlock()
if err != nil {
segment.deleting = false
return fmt.Errorf("delete acknowledged durable outbox wal segment: %w", err)
}
delete(w.segmentByID, segment.id)
for index, candidate := range w.segments {
if candidate != segment {
continue
}
w.segments = append(w.segments[:index], w.segments[index+1:]...)
break
}
return nil
}
func (w *durableOutboxWAL) recordStateLocked(ref outboxWALRecordRef) *outboxWALRecordState {
segment := w.segmentByID[ref.segmentID]
if segment == nil || ref.index < 0 || ref.index >= len(segment.records) {
return nil
}
return segment.records[ref.index]
}
func (w *durableOutboxWAL) setFatal(err error) {
if err == nil {
return
}
w.mu.Lock()
if w.fatalErr == nil {
w.fatalErr = err
}
w.mu.Unlock()
}
func encodeOutboxWALFrame(payload []byte) []byte {
frame := make([]byte, outboxWALHeaderSize+len(payload))
binary.BigEndian.PutUint32(frame[0:4], outboxWALMagic)
binary.BigEndian.PutUint32(frame[4:8], uint32(len(payload)))
binary.BigEndian.PutUint32(frame[8:12], crc32.ChecksumIEEE(payload))
copy(frame[outboxWALHeaderSize:], payload)
return frame
}
func outboxWALFileName(id uint64) string {
return fmt.Sprintf("outbox-%020d.wal", id)
}
func parseOutboxWALFileName(name string) (uint64, error) {
if !strings.HasPrefix(name, "outbox-") || !strings.HasSuffix(name, ".wal") {
return 0, fmt.Errorf("invalid durable outbox wal file name %q", name)
}
value := strings.TrimSuffix(strings.TrimPrefix(name, "outbox-"), ".wal")
id, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, fmt.Errorf("parse durable outbox wal file name %q: %w", name, err)
}
return id, nil
}
func syncDirectory(dir string) error {
handle, err := os.Open(dir)
if err != nil {
return err
}
syncErr := handle.Sync()
closeErr := handle.Close()
return errors.Join(syncErr, closeErr)
}
func minDuration(left, right time.Duration) time.Duration {
if left <= 0 {
return right
}
if right <= 0 || left < right {
return left
}
return right
}

229
go/vehicle-gateway/internal/eventbus/durable_outbox_wal_test.go Normal file
View File

@@ -0,0 +1,229 @@
package eventbus
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
func TestDurableOutboxWALConcurrentGroupCommitRecoversEveryRecord(t *testing.T) {
dir := t.TempDir()
wal, err := newDurableOutboxWAL(durableOutboxWALConfig{
Directory: dir,
SyncWrites: true,
CommitBatch: 64,
CommitInterval: 5 * time.Millisecond,
})
if err != nil {
t.Fatalf("new WAL: %v", err)
}
const total = 256
refs := make(chan outboxWALRecordRef, total)
errs := make(chan error, total)
var workers sync.WaitGroup
for i := 0; i < total; i++ {
workers.Add(1)
go func(sequence int) {
defer workers.Done()
record := walTestRecord(sequence)
stored, appendErr := wal.Append(context.Background(), record)
if appendErr != nil {
errs <- appendErr
return
}
refs <- stored.Ref
}(i)
}
workers.Wait()
close(errs)
for appendErr := range errs {
t.Fatalf("concurrent append: %v", appendErr)
}
close(refs)
for ref := range refs {
wal.Release(ref)
}
if got, _ := wal.Stats(); got != total {
t.Fatalf("backlog before restart = %d, want %d", got, total)
}
if err := wal.Close(); err != nil {
t.Fatalf("close first WAL: %v", err)
}
recovered, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir, SyncWrites: true})
if err != nil {
t.Fatalf("reopen WAL: %v", err)
}
records, err := recovered.ClaimPending(total + 1)
if err != nil {
t.Fatalf("claim recovered records: %v", err)
}
if len(records) != total {
t.Fatalf("recovered records = %d, want %d", len(records), total)
}
seen := make(map[string]struct{}, total)
for _, record := range records {
seen[record.Record.Envelope.EventID] = struct{}{}
if err := recovered.Ack(record.Ref); err != nil {
t.Fatalf("ack recovered record: %v", err)
}
}
if len(seen) != total {
t.Fatalf("unique recovered event ids = %d, want %d", len(seen), total)
}
if got, _ := recovered.Stats(); got != 0 {
t.Fatalf("backlog after ack = %d, want 0", got)
}
if err := recovered.Close(); err != nil {
t.Fatalf("close recovered WAL: %v", err)
}
}
func TestDurableOutboxWALTruncatesIncompleteTrailingFrame(t *testing.T) {
dir := t.TempDir()
first := encodedWALTestFrame(t, 1)
second := encodedWALTestFrame(t, 2)
path := filepath.Join(dir, outboxWALFileName(1))
payload := append(append([]byte{}, first...), second[:len(second)/2]...)
if err := os.WriteFile(path, payload, 0o640); err != nil {
t.Fatalf("write incomplete WAL: %v", err)
}
wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir, SyncWrites: true})
if err != nil {
t.Fatalf("recover incomplete WAL: %v", err)
}
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat recovered segment: %v", err)
}
if got, want := info.Size(), int64(len(first)); got != want {
t.Fatalf("truncated size = %d, want %d", got, want)
}
if got, _ := wal.Stats(); got != 1 {
t.Fatalf("recovered backlog = %d, want 1", got)
}
if err := wal.Close(); err != nil {
t.Fatalf("close WAL: %v", err)
}
}
func TestDurableOutboxWALRejectsChecksumCorruption(t *testing.T) {
dir := t.TempDir()
frame := encodedWALTestFrame(t, 1)
frame[len(frame)-1] ^= 0xff
path := filepath.Join(dir, outboxWALFileName(1))
if err := os.WriteFile(path, frame, 0o640); err != nil {
t.Fatalf("write corrupt WAL: %v", err)
}
_, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: dir})
if err == nil || !strings.Contains(err.Error(), "checksum mismatch") {
t.Fatalf("corrupt WAL error = %v", err)
}
}
func TestDurableOutboxWALAckDeletesClosedSegment(t *testing.T) {
dir := t.TempDir()
frameSize := int64(len(encodedWALTestFrame(t, 1)))
wal, err := newDurableOutboxWAL(durableOutboxWALConfig{
Directory: dir,
SegmentBytes: frameSize,
CommitBatch: 1,
})
if err != nil {
t.Fatalf("new WAL: %v", err)
}
first, err := wal.Append(context.Background(), walTestRecord(1))
if err != nil {
t.Fatalf("append first: %v", err)
}
firstPath := filepath.Join(dir, outboxWALFileName(first.Ref.segmentID))
second, err := wal.Append(context.Background(), walTestRecord(2))
if err != nil {
t.Fatalf("append second: %v", err)
}
if first.Ref.segmentID == second.Ref.segmentID {
t.Fatal("second append should rotate to a new segment")
}
if err := wal.Ack(first.Ref); err != nil {
t.Fatalf("ack first: %v", err)
}
if _, err := os.Stat(firstPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("closed acknowledged segment still exists: %v", err)
}
wal.Release(second.Ref)
if err := wal.Close(); err != nil {
t.Fatalf("close WAL: %v", err)
}
}
func TestDurableOutboxWALClaimIsBoundedAndNotDuplicated(t *testing.T) {
wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: t.TempDir(), CommitBatch: 1})
if err != nil {
t.Fatalf("new WAL: %v", err)
}
for i := 0; i < 10; i++ {
stored, err := wal.Append(context.Background(), walTestRecord(i))
if err != nil {
t.Fatalf("append %d: %v", i, err)
}
wal.Release(stored.Ref)
}
first, err := wal.ClaimPending(3)
if err != nil || len(first) != 3 {
t.Fatalf("first claim = %d, error = %v", len(first), err)
}
second, err := wal.ClaimPending(3)
if err != nil || len(second) != 3 {
t.Fatalf("second claim = %d, error = %v", len(second), err)
}
claimed := map[outboxWALRecordRef]struct{}{}
for _, record := range append(first, second...) {
if _, duplicate := claimed[record.Ref]; duplicate {
t.Fatalf("record claimed twice: %#v", record.Ref)
}
claimed[record.Ref] = struct{}{}
wal.Release(record.Ref)
}
if err := wal.Close(); err != nil {
t.Fatalf("close WAL: %v", err)
}
}
func TestDurableOutboxWALRejectsAppendAfterClose(t *testing.T) {
wal, err := newDurableOutboxWAL(durableOutboxWALConfig{Directory: t.TempDir()})
if err != nil {
t.Fatalf("new WAL: %v", err)
}
if err := wal.Close(); err != nil {
t.Fatalf("close WAL: %v", err)
}
_, err = wal.Append(context.Background(), walTestRecord(1))
if !errors.Is(err, ErrDurableOutboxWALClosed) {
t.Fatalf("append after close error = %v", err)
}
}
func walTestRecord(sequence int) durableRecord {
env := normalizeDurableEnvelope(durableTestEnvelope())
env.EventID = "wal-event-" + time.Unix(0, int64(sequence)+1).UTC().Format("150405.000000000")
env.Sequence = uint16(sequence)
return durableRecord{Kind: "raw", Envelope: env}
}
func encodedWALTestFrame(t *testing.T, sequence int) []byte {
t.Helper()
payload, err := json.Marshal(walTestRecord(sequence))
if err != nil {
t.Fatalf("marshal WAL test record: %v", err)
}
return encodeOutboxWALFrame(payload)
}

168
go/vehicle-gateway/internal/eventbus/durable_sink.go
View File

@@ -13,16 +13,21 @@ import (
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
type DurableConfig struct {
Directory string
ReplayBatchSize int
Metrics *metrics.Registry
Name string
}
type DurableSink struct {
delegate Sink
dir string
metrics *metrics.Registry
name string
mu sync.Mutex
seq uint64
@@ -50,12 +55,16 @@ func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
if delegate == nil {
panic("durable delegate sink must not be nil")
}
return &DurableSink{
s := &DurableSink{
delegate: delegate,
dir: strings.TrimSpace(cfg.Directory),
metrics: cfg.Metrics,
name: durableMetricName(cfg.Name),
rawPending: map[string]struct{}{},
replayBatchSize: cfg.ReplayBatchSize,
}
s.recordBacklogAfterReplay()
return s
}
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -96,16 +105,29 @@ func (s *DurableSink) ReplayOnce(ctx context.Context) error {
func (s *DurableSink) replay(ctx context.Context, limit int) error {
files, err := durableFiles(s.dir, limit)
if err != nil {
s.recordReplay("list_error", 0)
return err
}
s.recordBacklogAfterReplay()
records := make([]durableRecordFile, 0, len(files))
for _, file := range files {
record, err := readDurableRecord(file)
if err != nil {
return err
s.recordReplay("read_error", 1)
if quarantineErr := quarantineDurableFile(file); quarantineErr != nil {
s.recordReplay("quarantine_error", 1)
return fmt.Errorf("quarantine unreadable durable record %s: read error: %w; quarantine error: %v", file, err, quarantineErr)
}
s.recordReplay("quarantined", 1)
continue
}
records = append(records, durableRecordFile{path: file, record: record})
}
if len(records) == 0 {
s.recordReplay("empty", 0)
s.recordBacklogAfterReplay()
return nil
}
sortDurableRecords(records)
if publisher, ok := s.delegate.(recordPublishingSink); ok {
durableRecords := make([]durableRecord, 0, len(records))
@@ -113,29 +135,41 @@ func (s *DurableSink) replay(ctx context.Context, limit int) error {
durableRecords = append(durableRecords, item.record)
}
if err := publisher.PublishRecords(ctx, durableRecords); err != nil {
s.recordReplay("publish_error", len(records))
s.recordReplayRecords(records, "publish_error")
return err
}
for _, item := range records {
if err := os.Remove(item.path); err != nil {
s.recordReplay("delete_error", len(records))
return err
}
if item.record.Kind == "raw" {
s.clearRawPending(item.record.Envelope)
}
}
s.recordReplay("ok", len(records))
s.recordReplayRecords(records, "ok")
s.recordBacklogAfterReplay()
return nil
}
for _, item := range records {
if err := s.publishRecord(ctx, item.record); err != nil {
s.recordReplay("publish_error", len(records))
s.recordReplayRecord(item.record, "publish_error")
return err
}
if err := os.Remove(item.path); err != nil {
s.recordReplay("delete_error", len(records))
return err
}
if item.record.Kind == "raw" {
s.clearRawPending(item.record.Envelope)
}
s.recordReplayRecord(item.record, "ok")
}
s.recordReplay("ok", len(records))
s.recordBacklogAfterReplay()
return nil
}
@@ -210,7 +244,7 @@ func errUnknownRecordKind(kind string) error {
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
if s.dir == "" {
return fmt.Errorf("durable spool directory is empty")
return s.spoolError(kind, fmt.Errorf("durable spool directory is empty"))
}
if env.EventID == "" {
env.EventID = env.StableEventID()
@@ -219,19 +253,24 @@ func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
env.ParseStatus = envelope.ParseOK
}
if err := os.MkdirAll(s.dir, 0o750); err != nil {
return err
return s.spoolError(kind, err)
}
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
if err != nil {
return err
return s.spoolError(kind, err)
}
name := s.nextFileName(env, kind)
path := filepath.Join(s.dir, name)
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
return err
return s.spoolError(kind, err)
}
return os.Rename(tmp, path)
if err := os.Rename(tmp, path); err != nil {
return s.spoolError(kind, err)
}
s.recordSpool(kind, "ok")
s.recordBacklogAfterReplay()
return nil
}
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
@@ -273,6 +312,16 @@ func readDurableRecord(path string) (durableRecord, error) {
return record, json.Unmarshal(payload, &record)
}
func quarantineDurableFile(path string) error {
target := path + ".bad"
if _, err := os.Stat(target); err == nil {
target = fmt.Sprintf("%s.%d.bad", path, time.Now().UnixNano())
} else if !os.IsNotExist(err) {
return err
}
return os.Rename(path, target)
}
func durableFiles(dir string, limit int) ([]string, error) {
if limit > 0 {
handle, err := os.Open(dir)
@@ -330,3 +379,108 @@ func durableFilesFromReader(dir string, limit int, reader durableNameReader) ([]
func errorsIsEOF(err error) bool {
return err == io.EOF
}
func durableMetricName(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return "default"
}
return name
}
func (s *DurableSink) spoolError(kind string, err error) error {
s.recordSpool(kind, "error")
return err
}
func (s *DurableSink) recordSpool(kind string, status string) {
if s == nil || s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{
"name": s.name,
"kind": kind,
"status": status,
})
}
func (s *DurableSink) recordReplay(status string, records int) {
if s == nil || s.metrics == nil {
return
}
labels := metrics.Labels{"name": s.name, "status": status}
s.metrics.IncCounter("vehicle_durable_spool_replay_total", labels)
if records > 0 {
s.metrics.AddCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
"name": s.name,
"kind": "all",
"status": status,
}, float64(records))
}
}
func (s *DurableSink) recordReplayRecords(records []durableRecordFile, status string) {
for _, item := range records {
s.recordReplayRecord(item.record, status)
}
}
func (s *DurableSink) recordReplayRecord(record durableRecord, status string) {
if s == nil || s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
"name": s.name,
"kind": record.Kind,
"status": status,
})
}
func (s *DurableSink) recordBacklogAfterReplay() {
if s == nil || s.metrics == nil {
return
}
if strings.TrimSpace(s.dir) == "" {
s.recordBacklog(0, 0)
return
}
files, oldestAge, err := durableBacklogStats(s.dir, time.Now())
if err != nil {
return
}
s.recordBacklog(files, oldestAge)
}
func (s *DurableSink) recordBacklog(files int, oldestAge time.Duration) {
if s == nil || s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_durable_spool_backlog_files", metrics.Labels{"name": s.name}, float64(files))
ageSeconds := 0.0
if files > 0 && oldestAge > 0 {
ageSeconds = oldestAge.Seconds()
}
s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", metrics.Labels{"name": s.name}, ageSeconds)
}
func durableBacklogStats(dir string, now time.Time) (count int, oldestAge time.Duration, err error) {
files, err := durableFiles(dir, 0)
if err != nil {
return 0, 0, err
}
for _, file := range files {
info, statErr := os.Stat(file)
if statErr != nil {
return 0, 0, statErr
}
age := now.Sub(info.ModTime())
if age < 0 {
age = 0
}
if count == 0 || age > oldestAge {
oldestAge = age
}
count++
}
return count, oldestAge, nil
}

152
go/vehicle-gateway/internal/eventbus/durable_sink_test.go
View File

@@ -8,8 +8,10 @@ import (
"os"
"path/filepath"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
@@ -167,6 +169,156 @@ func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) {
}
}
func TestDurableSinkRecordsSpoolAndReplayMetrics(t *testing.T) {
dir := t.TempDir()
registry := metrics.NewRegistry()
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
env := durableTestEnvelope()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
text := registry.Render()
for _, want := range []string{
`vehicle_durable_spool_records_total{kind="raw",name="nats",status="ok"} 1`,
`vehicle_durable_spool_backlog_files{name="nats"} 1`,
`vehicle_durable_spool_oldest_age_seconds{name="nats"}`,
} {
if !containsString(text, want) {
t.Fatalf("spool metric missing %s:\n%s", want, text)
}
}
delegate.rawErrors = nil
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
text = registry.Render()
for _, want := range []string{
`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
`vehicle_durable_spool_replay_records_total{kind="all",name="nats",status="ok"} 1`,
`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
`vehicle_durable_spool_backlog_files{name="nats"} 0`,
`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
} {
if !containsString(text, want) {
t.Fatalf("replay metric missing %s:\n%s", want, text)
}
}
}
func TestDurableSinkInitializesBacklogMetrics(t *testing.T) {
dir := t.TempDir()
registry := metrics.NewRegistry()
_ = NewDurableSink(&scriptedSink{}, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
text := registry.Render()
for _, want := range []string{
`vehicle_durable_spool_backlog_files{name="nats"} 0`,
`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
} {
if !containsString(text, want) {
t.Fatalf("initial spool metric missing %s:\n%s", want, text)
}
}
}
func TestDurableSinkRecordsReplayPublishErrorMetrics(t *testing.T) {
dir := t.TempDir()
registry := metrics.NewRegistry()
env := durableTestEnvelope()
writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "kafka"})
if err := sink.ReplayOnce(context.Background()); err == nil {
t.Fatal("ReplayOnce() error = nil, want delegate publish error")
}
text := registry.Render()
for _, want := range []string{
`vehicle_durable_spool_replay_total{name="kafka",status="publish_error"} 1`,
`vehicle_durable_spool_replay_records_total{kind="all",name="kafka",status="publish_error"} 1`,
`vehicle_durable_spool_replay_records_total{kind="raw",name="kafka",status="publish_error"} 1`,
`vehicle_durable_spool_backlog_files{name="kafka"} 1`,
} {
if !containsString(text, want) {
t.Fatalf("replay error metric missing %s:\n%s", want, text)
}
}
if files := spoolFiles(t, dir); len(files) != 1 {
t.Fatalf("failed replay should keep spool file, files=%#v", files)
}
}
func TestDurableBacklogStatsCountsAllFilesAndOldestAge(t *testing.T) {
dir := t.TempDir()
env := durableTestEnvelope()
now := time.Date(2026, 7, 12, 15, 0, 0, 0, time.UTC)
oldFile := filepath.Join(dir, "0001-raw.json")
newFile := filepath.Join(dir, "0002-fields.json")
writeDurableRecord(t, oldFile, durableRecord{Kind: "raw", Envelope: env})
writeDurableRecord(t, newFile, durableRecord{Kind: "fields", Envelope: env})
if err := os.Chtimes(oldFile, now.Add(-10*time.Minute), now.Add(-10*time.Minute)); err != nil {
t.Fatalf("chtimes old file: %v", err)
}
if err := os.Chtimes(newFile, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil {
t.Fatalf("chtimes new file: %v", err)
}
count, oldestAge, err := durableBacklogStats(dir, now)
if err != nil {
t.Fatalf("durableBacklogStats() error = %v", err)
}
if count != 2 {
t.Fatalf("count = %d, want 2", count)
}
if oldestAge != 10*time.Minute {
t.Fatalf("oldestAge = %s, want 10m", oldestAge)
}
}
func TestDurableSinkQuarantinesBadRecordAndReplaysRemainingFiles(t *testing.T) {
dir := t.TempDir()
registry := metrics.NewRegistry()
env := durableTestEnvelope()
if err := os.WriteFile(filepath.Join(dir, "0001-bad.json"), []byte("{bad json"), 0o640); err != nil {
t.Fatalf("write bad durable record: %v", err)
}
writeDurableRecord(t, filepath.Join(dir, "0002-raw.json"), durableRecord{Kind: "raw", Envelope: env})
delegate := &scriptedSink{}
sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
if err := sink.ReplayOnce(context.Background()); err != nil {
t.Fatalf("ReplayOnce() error = %v", err)
}
if delegate.rawCalls != 1 {
t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
}
if files := spoolFiles(t, dir); len(files) != 0 {
t.Fatalf("normal spool files after replay = %#v, want none", files)
}
badFiles, err := filepath.Glob(filepath.Join(dir, "*.bad"))
if err != nil {
t.Fatalf("glob bad files: %v", err)
}
if len(badFiles) != 1 {
t.Fatalf("bad files = %#v, want one quarantined file", badFiles)
}
text := registry.Render()
for _, want := range []string{
`vehicle_durable_spool_replay_total{name="nats",status="read_error"} 1`,
`vehicle_durable_spool_replay_total{name="nats",status="quarantined"} 1`,
`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
`vehicle_durable_spool_backlog_files{name="nats"} 0`,
} {
if !containsString(text, want) {
t.Fatalf("quarantine metric missing %s:\n%s", want, text)
}
}
}
func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
reader := &fakeNameReader{
batches: [][]string{

30
go/vehicle-gateway/internal/eventbus/kafka_retry.go Normal file
View File

@@ -0,0 +1,30 @@
package eventbus
import "github.com/segmentio/kafka-go"
// MessagesAfterCommittedPrefixes keeps every fetched message that is not
// covered by the highest committed offset for its topic partition. This is
// used when a batch partially succeeds: later poison or valid messages must
// remain in memory until the failed offset ahead of them is durably handled.
func MessagesAfterCommittedPrefixes(messages []kafka.Message, committed []kafka.Message) []kafka.Message {
type partitionKey struct {
topic string
partition int
}
highest := make(map[partitionKey]int64, len(committed))
for _, message := range committed {
key := partitionKey{topic: message.Topic, partition: message.Partition}
if offset, ok := highest[key]; !ok || message.Offset > offset {
highest[key] = message.Offset
}
}
remaining := make([]kafka.Message, 0, len(messages))
for _, message := range messages {
key := partitionKey{topic: message.Topic, partition: message.Partition}
if offset, ok := highest[key]; ok && message.Offset <= offset {
continue
}
remaining = append(remaining, message)
}
return remaining
}

34
go/vehicle-gateway/internal/eventbus/kafka_retry_test.go Normal file
View File

@@ -0,0 +1,34 @@
package eventbus
import (
"testing"
"github.com/segmentio/kafka-go"
)
func TestMessagesAfterCommittedPrefixesKeepsPartitionGaps(t *testing.T) {
messages := []kafka.Message{
{Topic: "raw", Partition: 0, Offset: 10},
{Topic: "raw", Partition: 1, Offset: 20},
{Topic: "raw", Partition: 0, Offset: 11},
{Topic: "raw", Partition: 1, Offset: 21},
{Topic: "raw", Partition: 0, Offset: 12},
}
committed := []kafka.Message{
{Topic: "raw", Partition: 0, Offset: 10},
{Topic: "raw", Partition: 1, Offset: 21},
}
remaining := MessagesAfterCommittedPrefixes(messages, committed)
if len(remaining) != 2 || remaining[0].Partition != 0 || remaining[0].Offset != 11 || remaining[1].Offset != 12 {
t.Fatalf("remaining = %#v", remaining)
}
}
func TestMessagesAfterCommittedPrefixesWithoutCommitKeepsWholeBatch(t *testing.T) {
messages := []kafka.Message{{Topic: "raw", Partition: 0, Offset: 10}}
remaining := MessagesAfterCommittedPrefixes(messages, nil)
if len(remaining) != 1 || remaining[0].Offset != 10 {
t.Fatalf("remaining = %#v", remaining)
}
}

36
go/vehicle-gateway/internal/eventbus/kafka_sink.go
View File

@@ -41,16 +41,38 @@ func NewKafkaSink(cfg KafkaConfig) (*KafkaSink, error) {
if len(cfg.Brokers) == 0 {
return nil, errors.New("kafka brokers are required")
}
if err := ValidateKafkaConfig(cfg); err != nil {
return nil, err
}
rawTopics, fieldsTopics := kafkaTopicMaps(cfg)
return newKafkaSinkWithWriter(&kafka.Writer{
Addr: kafka.TCP(cfg.Brokers...),
Balancer: &kafka.Hash{},
AllowAutoTopicCreation: false,
RequiredAcks: kafka.RequireAll,
Async: false,
}, cfg), nil
}, KafkaConfig{
RawTopics: rawTopics,
FieldsTopics: fieldsTopics,
UnifiedTopic: cfg.UnifiedTopic,
}), nil
}
func ValidateKafkaConfig(cfg KafkaConfig) error {
rawTopics, fieldsTopics := kafkaTopicMaps(cfg)
return topics.ValidateKafkaRawFields(protocolTopicLabels(rawTopics), protocolTopicLabels(fieldsTopics))
}
func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
rawTopics, fieldsTopics := kafkaTopicMaps(cfg)
unifiedTopic := cfg.UnifiedTopic
if unifiedTopic == "" {
unifiedTopic = topics.Unified
}
return &KafkaSink{writer: writer, rawTopics: rawTopics, fieldsTopics: fieldsTopics, unifiedTopic: unifiedTopic}
}
func kafkaTopicMaps(cfg KafkaConfig) (map[envelope.Protocol]string, map[envelope.Protocol]string) {
rawTopics := map[envelope.Protocol]string{
envelope.ProtocolGB32960: topics.RawGB32960,
envelope.ProtocolJT808: topics.RawJT808,
@@ -71,11 +93,15 @@ func newKafkaSinkWithWriter(writer kafkaWriter, cfg KafkaConfig) *KafkaSink {
fieldsTopics[protocol] = topic
}
}
unifiedTopic := cfg.UnifiedTopic
if unifiedTopic == "" {
unifiedTopic = topics.Unified
return rawTopics, fieldsTopics
}
func protocolTopicLabels(values map[envelope.Protocol]string) map[string]string {
out := make(map[string]string, len(values))
for protocol, topic := range values {
out[string(protocol)] = topic
}
return &KafkaSink{writer: writer, rawTopics: rawTopics, fieldsTopics: fieldsTopics, unifiedTopic: unifiedTopic}
return out
}
func (s *KafkaSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {

44
go/vehicle-gateway/internal/eventbus/kafka_sink_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"reflect"
"strings"
"testing"
"github.com/segmentio/kafka-go"
@@ -112,6 +113,49 @@ func TestNewKafkaSinkUsesProductionDeliveryGuarantees(t *testing.T) {
}
}
func TestValidateKafkaConfigRejectsRawFieldsTopicOverlap(t *testing.T) {
err := ValidateKafkaConfig(KafkaConfig{
RawTopics: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1",
},
FieldsTopics: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1",
},
})
if err == nil {
t.Fatal("ValidateKafkaConfig() error = nil, want overlap rejection")
}
if !strings.Contains(err.Error(), "fields kafka topic") {
t.Fatalf("error = %q, want fields topic family hint", err)
}
}
func TestValidateKafkaConfigRejectsKnownProtocolTopicMismatch(t *testing.T) {
err := ValidateKafkaConfig(KafkaConfig{
RawTopics: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.raw.go.gb32960.v1",
},
})
if err == nil {
t.Fatal("ValidateKafkaConfig() error = nil, want raw protocol mismatch")
}
if !strings.Contains(err.Error(), "must match protocol") {
t.Fatalf("error = %q, want protocol mismatch hint", err)
}
err = ValidateKafkaConfig(KafkaConfig{
FieldsTopics: map[envelope.Protocol]string{
envelope.ProtocolYutongMQTT: "vehicle.fields.go.jt808.v1",
},
})
if err == nil {
t.Fatal("ValidateKafkaConfig() error = nil, want fields protocol mismatch")
}
if !strings.Contains(err.Error(), "must match protocol") {
t.Fatalf("error = %q, want protocol mismatch hint", err)
}
}
type recordingWriter struct {
messages []kafka.Message
writeCalls int

183
go/vehicle-gateway/internal/eventbus/nats_sink.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/nats-io/nats.go"
@@ -13,16 +14,19 @@ import (
)
type NATSConfig struct {
URL string
Name string
RawSubjects map[envelope.Protocol]string
FieldsSubjects map[envelope.Protocol]string
UnifiedSubject string
URL string
Name string
RawSubjects map[envelope.Protocol]string
FieldsSubjects map[envelope.Protocol]string
UnifiedSubject string
AsyncMaxPending int
AsyncAckTimeout time.Duration
}
type NATSSink struct {
conn *nats.Conn
publisher natsPublisher
asyncPublisher natsAsyncPublisher
rawSubjects map[envelope.Protocol]string
fieldsSubjects map[envelope.Protocol]string
unifiedSubject string
@@ -34,10 +38,18 @@ type natsPublisher interface {
Publish(context.Context, string, []byte, ...NATSPublishOption) error
}
type natsAsyncPublisher interface {
PublishAsync(string, []byte, ...NATSPublishOption) (nats.PubAckFuture, error)
}
func NewNATSSink(cfg NATSConfig) (*NATSSink, error) {
if cfg.URL == "" {
return nil, errors.New("nats url is required")
}
if err := ValidateNATSConfig(cfg); err != nil {
return nil, err
}
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
name := cfg.Name
if name == "" {
name = "lingniu-vehicle-gateway"
@@ -46,17 +58,58 @@ func NewNATSSink(cfg NATSConfig) (*NATSSink, error) {
if err != nil {
return nil, err
}
js, err := conn.JetStream()
var jsOptions []nats.JSOpt
if cfg.AsyncMaxPending > 0 {
jsOptions = append(jsOptions, nats.PublishAsyncMaxPending(cfg.AsyncMaxPending))
}
if cfg.AsyncAckTimeout > 0 {
jsOptions = append(jsOptions, nats.PublishAsyncTimeout(cfg.AsyncAckTimeout))
}
js, err := conn.JetStream(jsOptions...)
if err != nil {
conn.Close()
return nil, err
}
sink := newNATSSinkWithPublisher(natsJetStreamPublisher{js: js}, cfg)
publisher := natsJetStreamPublisher{js: js}
sink := newNATSSinkWithPublishers(publisher, publisher, NATSConfig{
RawSubjects: rawSubjects,
FieldsSubjects: fieldsSubjects,
UnifiedSubject: cfg.UnifiedSubject,
})
sink.conn = conn
return sink, nil
}
func ValidateNATSConfig(cfg NATSConfig) error {
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
raw := protocolTopicLabels(rawSubjects)
fields := protocolTopicLabels(fieldsSubjects)
if err := topics.ValidateKnownRawFieldsProtocols(raw, fields, "nats subject"); err != nil {
return err
}
return topics.ValidateRawFieldsDisjoint(raw, fields, "nats subject")
}
func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink {
return newNATSSinkWithPublishers(publisher, nil, cfg)
}
func newNATSSinkWithPublishers(publisher natsPublisher, asyncPublisher natsAsyncPublisher, cfg NATSConfig) *NATSSink {
rawSubjects, fieldsSubjects := natsSubjectMaps(cfg)
unifiedSubject := cfg.UnifiedSubject
if unifiedSubject == "" {
unifiedSubject = topics.Unified
}
return &NATSSink{
publisher: publisher,
asyncPublisher: asyncPublisher,
rawSubjects: rawSubjects,
fieldsSubjects: fieldsSubjects,
unifiedSubject: unifiedSubject,
}
}
func natsSubjectMaps(cfg NATSConfig) (map[envelope.Protocol]string, map[envelope.Protocol]string) {
rawSubjects := map[envelope.Protocol]string{
envelope.ProtocolGB32960: topics.RawGB32960,
envelope.ProtocolJT808: topics.RawJT808,
@@ -77,16 +130,7 @@ func newNATSSinkWithPublisher(publisher natsPublisher, cfg NATSConfig) *NATSSink
fieldsSubjects[protocol] = subject
}
}
unifiedSubject := cfg.UnifiedSubject
if unifiedSubject == "" {
unifiedSubject = topics.Unified
}
return &NATSSink{
publisher: publisher,
rawSubjects: rawSubjects,
fieldsSubjects: fieldsSubjects,
unifiedSubject: unifiedSubject,
}
return rawSubjects, fieldsSubjects
}
func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -94,14 +138,14 @@ func (s *NATSSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) e
if !ok || subject == "" {
return fmt.Errorf("raw subject not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, subject, env)
return s.publish(ctx, subject, "raw", env)
}
func (s *NATSSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
if s.unifiedSubject == "" {
return errors.New("unified subject is empty")
}
return s.publish(ctx, s.unifiedSubject, env)
return s.publish(ctx, s.unifiedSubject, "unified", env)
}
func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -109,7 +153,62 @@ func (s *NATSSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope
if !ok || subject == "" {
return fmt.Errorf("fields subject not configured for protocol %s", env.Protocol)
}
return s.publish(ctx, subject, env)
return s.publish(ctx, subject, "fields", env)
}
func (s *NATSSink) PublishRecords(ctx context.Context, records []durableRecord) error {
for _, record := range records {
subject, err := s.subjectForRecord(record)
if err != nil {
return err
}
if err := s.publish(ctx, subject, record.Kind, record.Envelope); err != nil {
return err
}
}
return nil
}
func (s *NATSSink) PublishRecordAsync(record durableRecord, complete func(error)) error {
if s == nil || s.asyncPublisher == nil {
return errors.New("nats async publisher is not configured")
}
if complete == nil {
return errors.New("nats async publish completion callback is required")
}
subject, err := s.subjectForRecord(record)
if err != nil {
return err
}
payload, err := record.Envelope.MarshalJSONBytes()
if err != nil {
return err
}
future, err := s.asyncPublisher.PublishAsync(
subject,
payload,
nats.MsgId(natsMessageID(record.Kind, subject, record.Envelope)),
)
if err != nil {
return err
}
go func() {
select {
case <-future.Ok():
complete(nil)
case asyncErr := <-future.Err():
if asyncErr == nil {
asyncErr = errors.New("nats async publish failed without error detail")
}
complete(asyncErr)
}
}()
return nil
}
func (s *NATSSink) ValidateRecord(record durableRecord) error {
_, err := s.subjectForRecord(record)
return err
}
func (s *NATSSink) Close() error {
@@ -121,12 +220,48 @@ func (s *NATSSink) Close() error {
return nil
}
func (s *NATSSink) publish(ctx context.Context, subject string, env envelope.FrameEnvelope) error {
func (s *NATSSink) publish(ctx context.Context, subject string, kind string, env envelope.FrameEnvelope) error {
payload, err := env.MarshalJSONBytes()
if err != nil {
return err
}
return s.publisher.Publish(ctx, subject, payload, nats.MsgId(env.StableEventID()))
return s.publisher.Publish(ctx, subject, payload, nats.MsgId(natsMessageID(kind, subject, env)))
}
func natsMessageID(kind string, subject string, env envelope.FrameEnvelope) string {
kind = strings.TrimSpace(kind)
if kind == "" {
kind = "unknown"
}
subject = strings.TrimSpace(subject)
if subject == "" {
subject = "unknown"
}
return kind + ":" + subject + ":" + env.StableEventID()
}
func (s *NATSSink) subjectForRecord(record durableRecord) (string, error) {
switch record.Kind {
case "raw":
subject, ok := s.rawSubjects[record.Envelope.Protocol]
if !ok || subject == "" {
return "", fmt.Errorf("raw subject not configured for protocol %s", record.Envelope.Protocol)
}
return subject, nil
case "unified":
if s.unifiedSubject == "" {
return "", errors.New("unified subject is empty")
}
return s.unifiedSubject, nil
case "fields":
subject, ok := s.fieldsSubjects[record.Envelope.Protocol]
if !ok || subject == "" {
return "", fmt.Errorf("fields subject not configured for protocol %s", record.Envelope.Protocol)
}
return subject, nil
default:
return "", errUnknownRecordKind(record.Kind)
}
}
type natsJetStreamPublisher struct {
@@ -137,3 +272,7 @@ func (p natsJetStreamPublisher) Publish(ctx context.Context, subject string, dat
_, err := p.js.Publish(subject, data, append(opts, nats.Context(ctx))...)
return err
}
func (p natsJetStreamPublisher) PublishAsync(subject string, data []byte, opts ...NATSPublishOption) (nats.PubAckFuture, error) {
return p.js.PublishAsync(subject, data, opts...)
}

222
go/vehicle-gateway/internal/eventbus/nats_sink_test.go
View File

@@ -3,7 +3,12 @@ package eventbus
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
)
@@ -57,6 +62,190 @@ func TestNATSSinkDefaultsToGoRawSubjects(t *testing.T) {
}
}
func TestNATSSinkPublishesDurableRecords(t *testing.T) {
publisher := &recordingNATSPublisher{}
sink := newNATSSinkWithPublisher(publisher, NATSConfig{
RawSubjects: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.raw.go.jt808.v1",
},
FieldsSubjects: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.fields.go.jt808.v1",
},
UnifiedSubject: "vehicle.event.go.unified.v1",
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
err := sink.PublishRecords(context.Background(), []durableRecord{
{Kind: "raw", Envelope: env},
{Kind: "fields", Envelope: env},
{Kind: "unified", Envelope: env},
})
if err != nil {
t.Fatalf("PublishRecords() error = %v", err)
}
if len(publisher.messages) != 3 {
t.Fatalf("published messages = %d, want 3", len(publisher.messages))
}
for i, want := range []string{
"vehicle.raw.go.jt808.v1",
"vehicle.fields.go.jt808.v1",
"vehicle.event.go.unified.v1",
} {
if got := publisher.messages[i].subject; got != want {
t.Fatalf("message %d subject = %q, want %q", i, got, want)
}
}
}
func TestNATSMessageIDSeparatesKindAndSubject(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", Sequence: 7}
raw := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env)
rawRetry := natsMessageID("raw", "vehicle.raw.go.jt808.v1", env)
fields := natsMessageID("fields", "vehicle.fields.go.jt808.v1", env)
unified := natsMessageID("unified", "vehicle.event.go.unified.v1", env)
rawOtherSubject := natsMessageID("raw", "vehicle.raw.go.gb32960.v1", env)
if raw != rawRetry {
t.Fatalf("same kind/subject/event id should be stable: %q vs %q", raw, rawRetry)
}
if raw == fields || raw == unified || raw == rawOtherSubject {
t.Fatalf("message ids should be unique per kind and subject: raw=%q fields=%q unified=%q rawOther=%q", raw, fields, unified, rawOtherSubject)
}
if !strings.Contains(raw, env.StableEventID()) {
t.Fatalf("message id %q should retain stable event id %q", raw, env.StableEventID())
}
}
func TestNATSSinkPublishRecordsRejectsUnknownKind(t *testing.T) {
sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{})
err := sink.PublishRecords(context.Background(), []durableRecord{
{Kind: "unknown", Envelope: envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808}},
})
if err == nil || !strings.Contains(err.Error(), "unknown durable record kind") {
t.Fatalf("PublishRecords() error = %v, want unknown kind", err)
}
}
func TestNATSSinkAsyncPublishCompletesOnlyAfterPubAck(t *testing.T) {
future := newTestPubAckFuture()
asyncPublisher := &recordingNATSAsyncPublisher{future: future}
sink := newNATSSinkWithPublishers(&recordingNATSPublisher{}, asyncPublisher, NATSConfig{})
record := durableRecord{Kind: "raw", Envelope: normalizeDurableEnvelope(durableTestEnvelope())}
completed := make(chan error, 1)
if err := sink.PublishRecordAsync(record, func(err error) { completed <- err }); err != nil {
t.Fatalf("PublishRecordAsync() error = %v", err)
}
select {
case err := <-completed:
t.Fatalf("completion fired before PubAck: %v", err)
case <-time.After(10 * time.Millisecond):
}
if got, want := asyncPublisher.subject, "vehicle.raw.go.jt808.v1"; got != want {
t.Fatalf("async subject = %q, want %q", got, want)
}
var decoded envelope.FrameEnvelope
if err := json.Unmarshal(asyncPublisher.data, &decoded); err != nil {
t.Fatalf("decode async payload: %v", err)
}
if decoded.EventID != record.Envelope.EventID {
t.Fatalf("async event id = %q, want %q", decoded.EventID, record.Envelope.EventID)
}
future.ok <- &nats.PubAck{Stream: "VEHICLE_RAW", Sequence: 1}
select {
case err := <-completed:
if err != nil {
t.Fatalf("completion error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("completion did not fire after PubAck")
}
}
func TestNATSSinkAsyncPublishPropagatesFutureError(t *testing.T) {
future := newTestPubAckFuture()
sink := newNATSSinkWithPublishers(
&recordingNATSPublisher{},
&recordingNATSAsyncPublisher{future: future},
NATSConfig{},
)
completed := make(chan error, 1)
errWant := errors.New("jetstream ack timeout")
if err := sink.PublishRecordAsync(durableRecord{
Kind: "raw",
Envelope: normalizeDurableEnvelope(durableTestEnvelope()),
}, func(err error) { completed <- err }); err != nil {
t.Fatalf("PublishRecordAsync() error = %v", err)
}
future.err <- errWant
select {
case err := <-completed:
if !errors.Is(err, errWant) {
t.Fatalf("completion error = %v, want %v", err, errWant)
}
case <-time.After(time.Second):
t.Fatal("completion did not fire after future error")
}
}
func TestNATSSinkValidatesDurableRecordBeforeOutboxPersistence(t *testing.T) {
sink := newNATSSinkWithPublisher(&recordingNATSPublisher{}, NATSConfig{})
err := sink.ValidateRecord(durableRecord{
Kind: "raw",
Envelope: envelope.FrameEnvelope{Protocol: envelope.Protocol("UNKNOWN")},
})
if err == nil || !strings.Contains(err.Error(), "raw subject not configured") {
t.Fatalf("ValidateRecord() error = %v", err)
}
}
func TestValidateNATSConfigRejectsRawFieldsSubjectOverlap(t *testing.T) {
err := ValidateNATSConfig(NATSConfig{
RawSubjects: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.same.jt808",
},
FieldsSubjects: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.same.jt808",
},
})
if err == nil {
t.Fatal("ValidateNATSConfig() error = nil, want overlap rejection")
}
if !strings.Contains(err.Error(), "nats subject") {
t.Fatalf("error = %q, want nats subject hint", err)
}
}
func TestValidateNATSConfigRejectsKnownProtocolSubjectMismatch(t *testing.T) {
err := ValidateNATSConfig(NATSConfig{
RawSubjects: map[envelope.Protocol]string{
envelope.ProtocolJT808: "vehicle.raw.go.gb32960.v1",
},
})
if err == nil {
t.Fatal("ValidateNATSConfig() error = nil, want raw protocol mismatch")
}
if !strings.Contains(err.Error(), "must match protocol") {
t.Fatalf("error = %q, want protocol mismatch hint", err)
}
err = ValidateNATSConfig(NATSConfig{
FieldsSubjects: map[envelope.Protocol]string{
envelope.ProtocolGB32960: "vehicle.fields.go.yutong-mqtt.v1",
},
})
if err == nil {
t.Fatal("ValidateNATSConfig() error = nil, want fields protocol mismatch")
}
if !strings.Contains(err.Error(), "must match protocol") {
t.Fatalf("error = %q, want protocol mismatch hint", err)
}
}
type recordingNATSPublisher struct {
messages []recordedNATSMessage
}
@@ -70,3 +259,36 @@ func (p *recordingNATSPublisher) Publish(_ context.Context, subject string, data
p.messages = append(p.messages, recordedNATSMessage{subject: subject, data: append([]byte(nil), data...)})
return nil
}
type recordingNATSAsyncPublisher struct {
subject string
data []byte
future nats.PubAckFuture
err error
}
func (p *recordingNATSAsyncPublisher) PublishAsync(subject string, data []byte, _ ...NATSPublishOption) (nats.PubAckFuture, error) {
p.subject = subject
p.data = append([]byte(nil), data...)
return p.future, p.err
}
type testPubAckFuture struct {
ok chan *nats.PubAck
err chan error
msg *nats.Msg
}
func newTestPubAckFuture() *testPubAckFuture {
return &testPubAckFuture{
ok: make(chan *nats.PubAck, 1),
err: make(chan error, 1),
msg: &nats.Msg{},
}
}
func (f *testPubAckFuture) Ok() <-chan *nats.PubAck { return f.ok }
func (f *testPubAckFuture) Err() <-chan error { return f.err }
func (f *testPubAckFuture) Msg() *nats.Msg { return f.msg }

293
go/vehicle-gateway/internal/eventbus/partitioned_async_sink.go Normal file
View File

@@ -0,0 +1,293 @@
package eventbus
import (
"context"
"sync"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
type PartitionedAsyncConfig struct {
RawQueueSize int
DerivedQueueSize int
RawWorkers int
DerivedWorkers int
EnqueueTimeout time.Duration
RawEnqueueTimeout time.Duration
DerivedEnqueueTimeout time.Duration
OperationTimeout time.Duration
OnError func(error)
Metrics *metrics.Registry
Name string
}
type PartitionedAsyncSink struct {
delegate Sink
rawJobs chan asyncJob
derivedJobs chan asyncJob
rawEnqueueTimeout time.Duration
derivedEnqueueTimeout time.Duration
timeout time.Duration
onError func(error)
metrics *metrics.Registry
name string
queueWait *metrics.RecentLatencyByKey
closeOnce sync.Once
closed chan struct{}
done chan struct{}
wg sync.WaitGroup
}
func NewPartitionedAsyncSink(delegate Sink, cfg PartitionedAsyncConfig) *PartitionedAsyncSink {
if delegate == nil {
panic("partitioned async delegate sink must not be nil")
}
if cfg.RawQueueSize <= 0 {
cfg.RawQueueSize = 100_000
}
if cfg.DerivedQueueSize <= 0 {
cfg.DerivedQueueSize = 50_000
}
if cfg.RawWorkers <= 0 {
cfg.RawWorkers = 4
}
if cfg.DerivedWorkers <= 0 {
cfg.DerivedWorkers = 2
}
enqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.EnqueueTimeout, time.Second)
rawEnqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.RawEnqueueTimeout, enqueueTimeout)
derivedEnqueueTimeout := normalizePartitionedEnqueueTimeout(cfg.DerivedEnqueueTimeout, enqueueTimeout)
if cfg.OperationTimeout <= 0 {
cfg.OperationTimeout = 30 * time.Second
}
if cfg.Name == "" {
cfg.Name = "partitioned-async"
}
s := &PartitionedAsyncSink{
delegate: delegate,
rawJobs: make(chan asyncJob, cfg.RawQueueSize),
derivedJobs: make(chan asyncJob, cfg.DerivedQueueSize),
rawEnqueueTimeout: rawEnqueueTimeout,
derivedEnqueueTimeout: derivedEnqueueTimeout,
timeout: cfg.OperationTimeout,
onError: cfg.OnError,
metrics: cfg.Metrics,
name: cfg.Name,
queueWait: metrics.NewRecentLatencyByKey(512),
closed: make(chan struct{}),
done: make(chan struct{}),
}
s.startWorkers("raw", s.rawJobs, cfg.RawWorkers)
s.startWorkers("derived", s.derivedJobs, cfg.DerivedWorkers)
s.recordWorkers("raw", cfg.RawWorkers)
s.recordWorkers("derived", cfg.DerivedWorkers)
s.recordQueueCapacity("raw", cap(s.rawJobs))
s.recordQueueCapacity("derived", cap(s.derivedJobs))
go func() {
s.wg.Wait()
close(s.done)
}()
return s
}
func (s *PartitionedAsyncSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
return s.enqueue(ctx, "raw", s.rawJobs, s.rawEnqueueTimeout, asyncJob{kind: "raw", env: env})
}
func (s *PartitionedAsyncSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
return s.enqueue(ctx, "derived", s.derivedJobs, s.derivedEnqueueTimeout, asyncJob{kind: "unified", env: env})
}
func (s *PartitionedAsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
return s.enqueue(ctx, "derived", s.derivedJobs, s.derivedEnqueueTimeout, asyncJob{kind: "fields", env: env})
}
func (s *PartitionedAsyncSink) Close() error {
s.closeOnce.Do(func() {
close(s.closed)
})
<-s.done
return s.delegate.Close()
}
func (s *PartitionedAsyncSink) startWorkers(queueName string, jobs <-chan asyncJob, workers int) {
s.wg.Add(workers)
for i := 0; i < workers; i++ {
go s.worker(queueName, jobs)
}
}
func (s *PartitionedAsyncSink) enqueue(ctx context.Context, queueName string, jobs chan<- asyncJob, enqueueTimeout time.Duration, job asyncJob) error {
select {
case <-s.closed:
s.recordEnqueue(job.kind, "closed")
return ErrAsyncSinkClosed
default:
}
var timeoutC <-chan time.Time
var timer *time.Timer
if enqueueTimeout > 0 {
timer = time.NewTimer(enqueueTimeout)
timeoutC = timer.C
defer timer.Stop()
}
job.enqueuedAt = time.Now()
select {
case jobs <- job:
s.recordEnqueue(job.kind, "queued")
s.recordQueueDepth(queueName)
return nil
case <-s.closed:
s.recordEnqueue(job.kind, "closed")
return ErrAsyncSinkClosed
case <-ctx.Done():
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth(queueName)
return ctx.Err()
case <-timeoutC:
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth(queueName)
return ErrAsyncSinkEnqueueTimeout
}
}
func normalizePartitionedEnqueueTimeout(value time.Duration, fallback time.Duration) time.Duration {
if value == 0 {
value = fallback
}
if value < 0 {
return 0
}
return value
}
func (s *PartitionedAsyncSink) worker(queueName string, jobs <-chan asyncJob) {
defer s.wg.Done()
for {
select {
case job := <-jobs:
s.publishJob(queueName, job)
case <-s.closed:
for {
select {
case job := <-jobs:
s.publishJob(queueName, job)
default:
return
}
}
}
}
}
func (s *PartitionedAsyncSink) publishJob(queueName string, job asyncJob) {
s.recordQueueDepth(queueName)
s.recordQueueWait(queueName, job)
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
started := time.Now()
var err error
switch job.kind {
case "raw":
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
}
cancel()
status := "ok"
if err != nil {
status = "error"
}
s.recordPublish(job.kind, status, time.Since(started))
if err != nil && s.onError != nil {
s.onError(err)
}
s.recordQueueDepth(queueName)
}
func (s *PartitionedAsyncSink) recordEnqueue(kind string, status string) {
if s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_async_sink_enqueue_total", metrics.Labels{
"sink": s.name,
"kind": kind,
"status": status,
})
}
func (s *PartitionedAsyncSink) recordPublish(kind string, status string, elapsed time.Duration) {
if s.metrics == nil {
return
}
labels := metrics.Labels{
"sink": s.name,
"kind": kind,
"status": status,
}
s.metrics.IncCounter("vehicle_async_sink_publish_total", labels)
elapsedMS := float64(elapsed.Milliseconds())
s.metrics.SetGauge("vehicle_async_sink_publish_duration_ms", labels, elapsedMS)
s.metrics.ObserveHistogram("vehicle_async_sink_publish_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS)
}
func (s *PartitionedAsyncSink) recordQueueDepth(queueName string) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_queue_depth", metrics.Labels{
"sink": s.name,
"queue": queueName,
}, float64(s.queueDepth(queueName)))
}
func (s *PartitionedAsyncSink) recordQueueCapacity(queueName string, value int) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_queue_capacity", metrics.Labels{
"sink": s.name,
"queue": queueName,
}, float64(value))
}
func (s *PartitionedAsyncSink) recordQueueWait(queueName string, job asyncJob) {
if s.metrics == nil || job.enqueuedAt.IsZero() {
return
}
elapsedMS := float64(time.Since(job.enqueuedAt)) / float64(time.Millisecond)
labels := metrics.Labels{
"sink": s.name,
"queue": queueName,
"kind": job.kind,
}
s.metrics.ObserveHistogram("vehicle_async_sink_queue_wait_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS)
p99, samples := s.queueWait.Observe(queueName+"\x00"+job.kind, elapsedMS)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_p99_ms", labels, p99)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_samples", labels, float64(samples))
}
func (s *PartitionedAsyncSink) recordWorkers(queueName string, workers int) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_workers", metrics.Labels{
"sink": s.name,
"queue": queueName,
}, float64(workers))
}
func (s *PartitionedAsyncSink) queueDepth(queueName string) int {
switch queueName {
case "raw":
return len(s.rawJobs)
case "derived":
return len(s.derivedJobs)
default:
return 0
}
}

230
go/vehicle-gateway/internal/eventbus/partitioned_async_sink_test.go Normal file
View File

@@ -0,0 +1,230 @@
package eventbus
import (
"context"
"errors"
"strings"
"testing"
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
func TestPartitionedAsyncSinkRawQueueIsIsolatedFromDerivedBacklog(t *testing.T) {
delegate := newBlockingDerivedSink()
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
RawQueueSize: 1,
DerivedQueueSize: 1,
RawWorkers: 1,
DerivedWorkers: 1,
EnqueueTimeout: 20 * time.Millisecond,
OperationTimeout: time.Second,
})
defer sink.Close()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("PublishFields() error = %v", err)
}
select {
case <-delegate.fieldsStarted:
case <-time.After(time.Second):
t.Fatal("delegate fields publish was not started")
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
start := time.Now()
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if elapsed := time.Since(start); elapsed > 50*time.Millisecond {
t.Fatalf("PublishRaw() blocked behind derived backlog for %s", elapsed)
}
delegate.release()
}
func TestPartitionedAsyncSinkRecordsPerQueueMetrics(t *testing.T) {
registry := metrics.NewRegistry()
delegate := newBlockingDerivedSink()
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
RawQueueSize: 2,
DerivedQueueSize: 3,
RawWorkers: 1,
DerivedWorkers: 1,
OperationTimeout: time.Second,
Metrics: registry,
Name: "nats",
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBSCB3D4R1234567"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("PublishRaw() error = %v", err)
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("PublishFields() error = %v", err)
}
delegate.release()
if err := sink.Close(); err != nil {
t.Fatalf("Close() error = %v", err)
}
text := registry.Render()
for _, want := range []string{
`vehicle_async_sink_queue_capacity{queue="raw",sink="nats"} 2`,
`vehicle_async_sink_queue_capacity{queue="derived",sink="nats"} 3`,
`vehicle_async_sink_workers{queue="raw",sink="nats"} 1`,
`vehicle_async_sink_workers{queue="derived",sink="nats"} 1`,
`vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`,
`vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`,
`vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`,
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="raw",sink="nats"} 1`,
`vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="raw",sink="nats"}`,
`vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="raw",sink="nats"} 1`,
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="fields",queue="derived",sink="nats"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("partitioned async metric missing %s:\n%s", want, text)
}
}
}
func TestPartitionedAsyncSinkUsesIndependentDerivedEnqueueTimeout(t *testing.T) {
delegate := newBlockingDerivedSink()
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
RawQueueSize: 1,
DerivedQueueSize: 1,
RawWorkers: 1,
DerivedWorkers: 1,
RawEnqueueTimeout: 200 * time.Millisecond,
DerivedEnqueueTimeout: 10 * time.Millisecond,
OperationTimeout: time.Second,
})
defer sink.Close()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("PublishFields() error = %v", err)
}
select {
case <-delegate.fieldsStarted:
case <-time.After(time.Second):
t.Fatal("delegate fields publish was not started")
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("first PublishUnified() error = %v", err)
}
start := time.Now()
err := sink.PublishUnified(context.Background(), env)
elapsed := time.Since(start)
if !errors.Is(err, ErrAsyncSinkEnqueueTimeout) {
t.Fatalf("second PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err)
}
if elapsed > 100*time.Millisecond {
t.Fatalf("derived enqueue timeout took %s, want quick failure", elapsed)
}
delegate.release()
}
func TestPartitionedAsyncSinkCloseUnblocksBlockedDerivedEnqueue(t *testing.T) {
delegate := newBlockingDerivedSink()
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
RawQueueSize: 1,
DerivedQueueSize: 1,
RawWorkers: 1,
DerivedWorkers: 1,
EnqueueTimeout: -1,
OperationTimeout: time.Second,
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("PublishFields() error = %v", err)
}
select {
case <-delegate.fieldsStarted:
case <-time.After(time.Second):
t.Fatal("delegate fields publish was not started")
}
if err := sink.PublishUnified(context.Background(), env); err != nil {
t.Fatalf("PublishUnified() error = %v", err)
}
publishErr := make(chan error, 1)
go func() {
publishErr <- sink.PublishUnified(context.Background(), env)
}()
closeErr := make(chan error, 1)
go func() {
closeErr <- sink.Close()
}()
select {
case err := <-publishErr:
if !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err)
}
case <-time.After(time.Second):
t.Fatal("blocked PublishUnified() was not released by Close")
}
delegate.release()
select {
case err := <-closeErr:
if err != nil {
t.Fatalf("Close() error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close() did not finish after delegate release")
}
}
type blockingDerivedSink struct {
fieldsStarted chan struct{}
releaseFields chan struct{}
}
func newBlockingDerivedSink() *blockingDerivedSink {
return &blockingDerivedSink{
fieldsStarted: make(chan struct{}),
releaseFields: make(chan struct{}),
}
}
func (s *blockingDerivedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *blockingDerivedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
return nil
}
func (s *blockingDerivedSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
s.signalFieldsStarted()
<-s.releaseFields
return nil
}
func (s *blockingDerivedSink) Close() error {
s.release()
return nil
}
func (s *blockingDerivedSink) signalFieldsStarted() {
select {
case <-s.fieldsStarted:
default:
close(s.fieldsStarted)
}
}
func (s *blockingDerivedSink) release() {
select {
case <-s.releaseFields:
default:
close(s.releaseFields)
}
}