shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
go/vehicle-gateway/internal/eventbus/async_sink.go
View File

@@ -13,6 +13,7 @@ import (
type AsyncConfig struct {
QueueSize int
Workers int
EnqueueTimeout time.Duration
OperationTimeout time.Duration
OnError func(error)
Metrics *metrics.Registry
@@ -20,12 +21,14 @@ type AsyncConfig struct {
}
type AsyncSink struct {
delegate Sink
jobs chan asyncJob
timeout time.Duration
onError func(error)
metrics *metrics.Registry
name string
delegate Sink
jobs chan asyncJob
enqueueTimeout time.Duration
timeout time.Duration
onError func(error)
metrics *metrics.Registry
name string
queueWait *metrics.RecentLatencyByKey
closeOnce sync.Once
closed chan struct{}
@@ -34,11 +37,13 @@ type AsyncSink struct {
}
type asyncJob struct {
kind string
env envelope.FrameEnvelope
kind string
env envelope.FrameEnvelope
enqueuedAt time.Time
}
var ErrAsyncSinkClosed = errors.New("async sink is closed")
var ErrAsyncSinkEnqueueTimeout = errors.New("async sink enqueue timeout")
var asyncSinkPublishDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
@@ -52,6 +57,12 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink {
if cfg.Workers <= 0 {
cfg.Workers = 1
}
if cfg.EnqueueTimeout == 0 {
cfg.EnqueueTimeout = time.Second
}
if cfg.EnqueueTimeout < 0 {
cfg.EnqueueTimeout = 0
}
if cfg.OperationTimeout <= 0 {
cfg.OperationTimeout = 30 * time.Second
}
@@ -59,19 +70,23 @@ func NewAsyncSink(delegate Sink, cfg AsyncConfig) *AsyncSink {
cfg.Name = "async"
}
s := &AsyncSink{
delegate: delegate,
jobs: make(chan asyncJob, cfg.QueueSize),
timeout: cfg.OperationTimeout,
onError: cfg.OnError,
metrics: cfg.Metrics,
name: cfg.Name,
closed: make(chan struct{}),
done: make(chan struct{}),
delegate: delegate,
jobs: make(chan asyncJob, cfg.QueueSize),
enqueueTimeout: cfg.EnqueueTimeout,
timeout: cfg.OperationTimeout,
onError: cfg.OnError,
metrics: cfg.Metrics,
name: cfg.Name,
queueWait: metrics.NewRecentLatencyByKey(512),
closed: make(chan struct{}),
done: make(chan struct{}),
}
s.wg.Add(cfg.Workers)
for i := 0; i < cfg.Workers; i++ {
go s.worker()
}
s.recordQueueCapacity()
s.recordWorkers("default", cfg.Workers)
go func() {
s.wg.Wait()
close(s.done)
@@ -94,7 +109,6 @@ func (s *AsyncSink) PublishFields(ctx context.Context, env envelope.FrameEnvelop
func (s *AsyncSink) Close() error {
s.closeOnce.Do(func() {
close(s.closed)
close(s.jobs)
})
<-s.done
return s.delegate.Close()
@@ -107,6 +121,14 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error {
return ErrAsyncSinkClosed
default:
}
var timeoutC <-chan time.Time
var timer *time.Timer
if s.enqueueTimeout > 0 {
timer = time.NewTimer(s.enqueueTimeout)
timeoutC = timer.C
defer timer.Stop()
}
job.enqueuedAt = time.Now()
select {
case s.jobs <- job:
s.recordEnqueue(job.kind, "queued")
@@ -119,37 +141,58 @@ func (s *AsyncSink) enqueue(ctx context.Context, job asyncJob) error {
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth()
return ctx.Err()
case <-timeoutC:
s.recordEnqueue(job.kind, "timeout")
s.recordQueueDepth()
return ErrAsyncSinkEnqueueTimeout
}
}
func (s *AsyncSink) worker() {
defer s.wg.Done()
for job := range s.jobs {
s.recordQueueDepth()
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
started := time.Now()
var err error
switch job.kind {
case "raw":
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
for {
select {
case job := <-s.jobs:
s.publishJob(job)
case <-s.closed:
for {
select {
case job := <-s.jobs:
s.publishJob(job)
default:
return
}
}
}
cancel()
status := "ok"
if err != nil {
status = "error"
}
s.recordPublish(job.kind, status, time.Since(started))
if err != nil && s.onError != nil {
s.onError(err)
}
s.recordQueueDepth()
}
}
func (s *AsyncSink) publishJob(job asyncJob) {
s.recordQueueDepth()
s.recordQueueWait("default", job)
ctx, cancel := context.WithTimeout(context.Background(), s.timeout)
started := time.Now()
var err error
switch job.kind {
case "raw":
err = s.delegate.PublishRaw(ctx, job.env)
case "unified":
err = s.delegate.PublishUnified(ctx, job.env)
case "fields":
err = s.delegate.PublishFields(ctx, job.env)
}
cancel()
status := "ok"
if err != nil {
status = "error"
}
s.recordPublish(job.kind, status, time.Since(started))
if err != nil && s.onError != nil {
s.onError(err)
}
s.recordQueueDepth()
}
func (s *AsyncSink) recordEnqueue(kind string, status string) {
if s.metrics == nil {
return
@@ -184,3 +227,38 @@ func (s *AsyncSink) recordQueueDepth() {
"sink": s.name,
}, float64(len(s.jobs)))
}
func (s *AsyncSink) recordQueueCapacity() {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_queue_capacity", metrics.Labels{
"sink": s.name,
}, float64(cap(s.jobs)))
}
func (s *AsyncSink) recordQueueWait(queueName string, job asyncJob) {
if s.metrics == nil || job.enqueuedAt.IsZero() {
return
}
elapsedMS := float64(time.Since(job.enqueuedAt)) / float64(time.Millisecond)
labels := metrics.Labels{
"sink": s.name,
"queue": queueName,
"kind": job.kind,
}
s.metrics.ObserveHistogram("vehicle_async_sink_queue_wait_duration_ms_histogram", labels, asyncSinkPublishDurationBucketsMS, elapsedMS)
p99, samples := s.queueWait.Observe(queueName+"\x00"+job.kind, elapsedMS)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_p99_ms", labels, p99)
s.metrics.SetGauge("vehicle_async_sink_queue_wait_recent_samples", labels, float64(samples))
}
func (s *AsyncSink) recordWorkers(queueName string, workers int) {
if s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_async_sink_workers", metrics.Labels{
"sink": s.name,
"queue": queueName,
}, float64(workers))
}