shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
go/vehicle-gateway/internal/eventbus/async_sink_test.go
View File

@@ -2,6 +2,7 @@ package eventbus
import (
"context"
"errors"
"strings"
"testing"
"time"
@@ -80,12 +81,17 @@ func TestAsyncSinkRecordsQueueMetrics(t *testing.T) {
for _, want := range []string{
`vehicle_async_sink_enqueue_total{kind="raw",sink="nats",status="queued"} 1`,
`vehicle_async_sink_enqueue_total{kind="fields",sink="nats",status="queued"} 1`,
`vehicle_async_sink_queue_capacity{sink="nats"} 2`,
`vehicle_async_sink_queue_depth{sink="nats"}`,
`vehicle_async_sink_workers{queue="default",sink="nats"} 1`,
`vehicle_async_sink_publish_total{kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_total{kind="fields",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_bucket{le="+Inf",kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_count{kind="raw",sink="nats",status="ok"} 1`,
`vehicle_async_sink_publish_duration_ms_histogram_sum{kind="raw",sink="nats",status="ok"}`,
`vehicle_async_sink_queue_wait_duration_ms_histogram_count{kind="raw",queue="default",sink="nats"} 1`,
`vehicle_async_sink_queue_wait_recent_p99_ms{kind="raw",queue="default",sink="nats"}`,
`vehicle_async_sink_queue_wait_recent_samples{kind="raw",queue="default",sink="nats"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("async sink metric missing %s:\n%s", want, text)
@@ -135,6 +141,145 @@ func TestAsyncSinkRecordsEnqueueTimeoutWhenQueueIsFull(t *testing.T) {
delegate.release()
}
func TestAsyncSinkEnqueueTimeoutDoesNotRequireCallerDeadline(t *testing.T) {
registry := metrics.NewRegistry()
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: 10 * time.Millisecond,
OperationTimeout: time.Second,
Metrics: registry,
Name: "nats",
})
defer sink.Close()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
if err := sink.PublishUnified(context.Background(), env); !errors.Is(err, ErrAsyncSinkEnqueueTimeout) {
t.Fatalf("third PublishUnified() error = %v, want ErrAsyncSinkEnqueueTimeout", err)
}
text := registry.Render()
for _, want := range []string{
`vehicle_async_sink_enqueue_total{kind="unified",sink="nats",status="timeout"} 1`,
`vehicle_async_sink_queue_depth{sink="nats"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("async sink enqueue timeout metric missing %s:\n%s", want, text)
}
}
delegate.release()
}
func TestAsyncSinkCloseDoesNotPanicWithConcurrentBlockedEnqueue(t *testing.T) {
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: 20 * time.Millisecond,
OperationTimeout: time.Second,
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
publishErr := make(chan error, 1)
go func() {
defer func() {
if recovered := recover(); recovered != nil {
publishErr <- errors.New("publish panicked")
}
}()
publishErr <- sink.PublishUnified(context.Background(), env)
}()
closeErr := make(chan error, 1)
go func() {
closeErr <- sink.Close()
}()
if err := <-publishErr; !errors.Is(err, ErrAsyncSinkEnqueueTimeout) && !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("concurrent PublishUnified() error = %v, want timeout or closed", err)
}
delegate.release()
if err := <-closeErr; err != nil {
t.Fatalf("Close() error = %v", err)
}
if err := sink.PublishRaw(context.Background(), env); !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("PublishRaw() after Close error = %v, want ErrAsyncSinkClosed", err)
}
}
func TestAsyncSinkCloseUnblocksPublishWhenEnqueueTimeoutIsDisabled(t *testing.T) {
delegate := newBlockingSink()
sink := NewAsyncSink(delegate, AsyncConfig{
QueueSize: 1,
Workers: 1,
EnqueueTimeout: -1,
OperationTimeout: time.Second,
})
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425"}
if err := sink.PublishRaw(context.Background(), env); err != nil {
t.Fatalf("first PublishRaw() error = %v", err)
}
select {
case <-delegate.rawStarted:
case <-time.After(time.Second):
t.Fatal("delegate raw publish was not started")
}
if err := sink.PublishFields(context.Background(), env); err != nil {
t.Fatalf("second PublishFields() error = %v", err)
}
publishErr := make(chan error, 1)
go func() {
publishErr <- sink.PublishUnified(context.Background(), env)
}()
closeErr := make(chan error, 1)
go func() {
closeErr <- sink.Close()
}()
select {
case err := <-publishErr:
if !errors.Is(err, ErrAsyncSinkClosed) {
t.Fatalf("blocked PublishUnified() error = %v, want ErrAsyncSinkClosed", err)
}
case <-time.After(time.Second):
t.Fatal("blocked PublishUnified() was not released by Close")
}
delegate.release()
select {
case err := <-closeErr:
if err != nil {
t.Fatalf("Close() error = %v", err)
}
case <-time.After(time.Second):
t.Fatal("Close() did not finish after delegate release")
}
}
type blockingSink struct {
rawStarted chan struct{}
releaseRaw chan struct{}