feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -0,0 +1,230 @@
|
||||
package eventbus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
)
|
||||
|
||||
func TestPartitionedAsyncSinkRawQueueIsIsolatedFromDerivedBacklog(t *testing.T) {
|
||||
delegate := newBlockingDerivedSink()
|
||||
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
|
||||
RawQueueSize: 1,
|
||||
DerivedQueueSize: 1,
|
||||
RawWorkers: 1,
|
||||
DerivedWorkers: 1,
|
||||
EnqueueTimeout: 20 * time.Millisecond,
|
||||
OperationTimeout: time.Second,
|
||||
})
|
||||
defer sink.Close()
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
|
||||
if err := sink.PublishFields(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishFields() error = %v", err)
|
||||
}
|
||||
select {
|
||||
case <-delegate.fieldsStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("delegate fields publish was not started")
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 50*time.Millisecond {
|
||||
t.Fatalf("PublishRaw() blocked behind derived backlog for %s", elapsed)
|
||||
}
|
||||
delegate.release()
|
||||
}
|
||||
|
||||
func TestPartitionedAsyncSinkRecordsPerQueueMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
delegate := newBlockingDerivedSink()
|
||||
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
|
||||
RawQueueSize: 2,
|
||||
DerivedQueueSize: 3,
|
||||
RawWorkers: 1,
|
||||
DerivedWorkers: 1,
|
||||
OperationTimeout: time.Second,
|
||||
Metrics: registry,
|
||||
Name: "nats",
|
||||
})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "LNBSCB3D4R1234567"}
|
||||
if err := sink.PublishRaw(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishRaw() error = %v", err)
|
||||
}
|
||||
if err := sink.PublishFields(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishFields() error = %v", err)
|
||||
}
|
||||
delegate.release()
|
||||
if err := sink.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_async_sink_queue_capacity{queue="raw",sink="nats"} 2`,
|
||||
`vehicle_async_sink_queue_capacity{queue="derived",sink="nats"} 3`,
|
||||
`vehicle_async_sink_workers{queue="raw",sink="nats"} 1`,
|
||||
`vehicle_async_sink_workers{queue="derived",sink="nats"} 1`,
|
||||
`vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`,
|
||||
`vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`,
|
||||
`vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`,
|
||||
`vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`,
|
||||
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="raw",sink="nats"} 1`,
|
||||
`vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="raw",sink="nats"}`,
|
||||
`vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="raw",sink="nats"} 1`,
|
||||
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="fields",queue="derived",sink="nats"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("partitioned async metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartitionedAsyncSinkUsesIndependentDerivedEnqueueTimeout(t *testing.T) {
|
||||
delegate := newBlockingDerivedSink()
|
||||
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
|
||||
RawQueueSize: 1,
|
||||
DerivedQueueSize: 1,
|
||||
RawWorkers: 1,
|
||||
DerivedWorkers: 1,
|
||||
RawEnqueueTimeout: 200 * time.Millisecond,
|
||||
DerivedEnqueueTimeout: 10 * time.Millisecond,
|
||||
OperationTimeout: time.Second,
|
||||
})
|
||||
defer sink.Close()
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
|
||||
if err := sink.PublishFields(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishFields() error = %v", err)
|
||||
}
|
||||
select {
|
||||
case <-delegate.fieldsStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("delegate fields publish was not started")
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("first PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
err := sink.PublishUnified(context.Background(), env)
|
||||
elapsed := time.Since(start)
|
||||
if !errors.Is(err, ErrAsyncSinkEnqueueTimeout) {
|
||||
t.Fatalf("second PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err)
|
||||
}
|
||||
if elapsed > 100*time.Millisecond {
|
||||
t.Fatalf("derived enqueue timeout took %s, want quick failure", elapsed)
|
||||
}
|
||||
delegate.release()
|
||||
}
|
||||
|
||||
func TestPartitionedAsyncSinkCloseUnblocksBlockedDerivedEnqueue(t *testing.T) {
|
||||
delegate := newBlockingDerivedSink()
|
||||
sink := NewPartitionedAsyncSink(delegate, PartitionedAsyncConfig{
|
||||
RawQueueSize: 1,
|
||||
DerivedQueueSize: 1,
|
||||
RawWorkers: 1,
|
||||
DerivedWorkers: 1,
|
||||
EnqueueTimeout: -1,
|
||||
OperationTimeout: time.Second,
|
||||
})
|
||||
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
|
||||
if err := sink.PublishFields(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishFields() error = %v", err)
|
||||
}
|
||||
select {
|
||||
case <-delegate.fieldsStarted:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("delegate fields publish was not started")
|
||||
}
|
||||
if err := sink.PublishUnified(context.Background(), env); err != nil {
|
||||
t.Fatalf("PublishUnified() error = %v", err)
|
||||
}
|
||||
|
||||
publishErr := make(chan error, 1)
|
||||
go func() {
|
||||
publishErr <- sink.PublishUnified(context.Background(), env)
|
||||
}()
|
||||
closeErr := make(chan error, 1)
|
||||
go func() {
|
||||
closeErr <- sink.Close()
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-publishErr:
|
||||
if !errors.Is(err, ErrAsyncSinkClosed) {
|
||||
t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("blocked PublishUnified() was not released by Close")
|
||||
}
|
||||
delegate.release()
|
||||
select {
|
||||
case err := <-closeErr:
|
||||
if err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("Close() did not finish after delegate release")
|
||||
}
|
||||
}
|
||||
|
||||
type blockingDerivedSink struct {
|
||||
fieldsStarted chan struct{}
|
||||
releaseFields chan struct{}
|
||||
}
|
||||
|
||||
func newBlockingDerivedSink() *blockingDerivedSink {
|
||||
return &blockingDerivedSink{
|
||||
fieldsStarted: make(chan struct{}),
|
||||
releaseFields: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
|
||||
s.signalFieldsStarted()
|
||||
<-s.releaseFields
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) Close() error {
|
||||
s.release()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) signalFieldsStarted() {
|
||||
select {
|
||||
case <-s.fieldsStarted:
|
||||
default:
|
||||
close(s.fieldsStarted)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *blockingDerivedSink) release() {
|
||||
select {
|
||||
case <-s.releaseFields:
|
||||
default:
|
||||
close(s.releaseFields)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user