shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(go): add realtime pipeline observability

Browse Source
This commit is contained in:
lingniu
2026-07-03 15:06:27 +08:00
parent 28c81611a1
commit b8299faefa
10 changed files with 636 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
go/vehicle-gateway/cmd/realtime-api/main.go
View File

@@ -57,8 +57,14 @@ func main() {
mux.Handle("/openapi.json", realtime.NewOpenAPIHandler())
mux.Handle("/swagger-ui/", realtime.NewSwaggerUIHandler())
mux.Handle("/api/realtime/vehicles/", realtime.NewHandler(repository))
mux.Handle("/api/realtime/online", realtime.NewOnlineIndexHandler(repository))
mux.Handle("/api/debug/pipeline", realtime.NewPipelineDebugHandler(repository))
closeStats := func() {}
var updater realtimeUpdater = fastRealtimeUpdater{repository: repository}
var updater realtimeUpdater = storeMetricUpdater{
store: "redis",
delegate: fastRealtimeUpdater{repository: repository},
registry: registry,
}
if dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")); dsn != "" {
db, err := sql.Open("mysql", dsn)
if err != nil {
@@ -84,16 +90,20 @@ func main() {
logger.Error("realtime snapshot mysql schema bootstrap failed", "error", err)
os.Exit(1)
}
mysqlUpdater := realtimeUpdater(snapshotWriter)
mysqlUpdater := realtimeUpdater(storeMetricUpdater{store: "mysql", delegate: snapshotWriter, registry: registry})
if env("MYSQL_REALTIME_ASYNC_ENABLED", "true") != "false" {
mysqlUpdater = newAsyncSecondaryRealtimeUpdater(
nil,
snapshotWriter,
storeMetricUpdater{store: "mysql", delegate: snapshotWriter, registry: registry},
envInt("MYSQL_REALTIME_ASYNC_QUEUE_SIZE", 20000),
envInt("MYSQL_REALTIME_ASYNC_WORKERS", 4),
registry,
)
}
updater = compositeRealtimeUpdater{primary: fastRealtimeUpdater{repository: repository}, secondary: mysqlUpdater}
updater = compositeRealtimeUpdater{
primary: storeMetricUpdater{store: "redis", delegate: fastRealtimeUpdater{repository: repository}, registry: registry},
secondary: mysqlUpdater,
}
logger.Info("realtime mysql snapshot enabled", "table", "vehicle_realtime_snapshot", "plate_binding_table", bindingTable, "plate_cache_ttl_seconds", envInt("PLATE_CACHE_TTL_SECONDS", 600))
}
mux.Handle("/api/stats/daily-metrics", stats.NewMetricHandler(stats.NewMetricRepository(db)))
@@ -227,6 +237,31 @@ func (u fastRealtimeUpdater) Update(ctx context.Context, env envelope.FrameEnvel
return u.repository.FastUpdate(ctx, env)
}
type storeMetricUpdater struct {
store string
delegate realtimeUpdater
registry *metrics.Registry
}
func (u storeMetricUpdater) Update(ctx context.Context, env envelope.FrameEnvelope) error {
if u.delegate == nil {
return nil
}
err := u.delegate.Update(ctx, env)
status := "ok"
if err != nil {
status = "error"
}
if u.registry != nil {
u.registry.IncCounter("vehicle_realtime_store_updates_total", metrics.Labels{
"store": u.store,
"protocol": string(env.Protocol),
"status": status,
})
}
return err
}
type compositeRealtimeUpdater struct {
primary realtimeUpdater
secondary realtimeUpdater
@@ -247,9 +282,10 @@ type asyncSecondaryRealtimeUpdater struct {
secondary realtimeUpdater
queue chan envelope.FrameEnvelope
cancel context.CancelFunc
registry *metrics.Registry
}
func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater, queueSize int, workers int) *asyncSecondaryRealtimeUpdater {
func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtimeUpdater, queueSize int, workers int, registry *metrics.Registry) *asyncSecondaryRealtimeUpdater {
if queueSize <= 0 {
queueSize = 20000
}
@@ -262,6 +298,7 @@ func newAsyncSecondaryRealtimeUpdater(primary realtimeUpdater, secondary realtim
secondary: secondary,
queue: make(chan envelope.FrameEnvelope, queueSize),
cancel: cancel,
registry: registry,
}
for i := 0; i < workers; i++ {
go updater.run(ctx)
@@ -280,8 +317,10 @@ func (u *asyncSecondaryRealtimeUpdater) Update(ctx context.Context, env envelope
}
select {
case u.queue <- env:
u.recordQueueMetric(env, "queued")
return nil
default:
u.recordQueueMetric(env, "dropped")
return nil
}
}
@@ -299,6 +338,17 @@ func (u *asyncSecondaryRealtimeUpdater) run(ctx context.Context) {
}
}
func (u *asyncSecondaryRealtimeUpdater) recordQueueMetric(env envelope.FrameEnvelope, status string) {
if u.registry == nil {
return
}
u.registry.IncCounter("vehicle_realtime_async_queue_total", metrics.Labels{
"store": "mysql",
"protocol": string(env.Protocol),
"status": status,
})
}
func (u *asyncSecondaryRealtimeUpdater) Close() {
if u.cancel != nil {
u.cancel()

35
go/vehicle-gateway/cmd/realtime-api/main_test.go
View File

@@ -102,7 +102,8 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"}
primary := &contextCheckingRealtimeUpdater{}
secondary := &blockingRealtimeUpdater{started: make(chan struct{}), release: make(chan struct{})}
updater := newAsyncSecondaryRealtimeUpdater(primary, secondary, 8, 1)
registry := metrics.NewRegistry()
updater := newAsyncSecondaryRealtimeUpdater(primary, secondary, 8, 1, registry)
defer updater.Close()
start := time.Now()
@@ -121,6 +122,26 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) {
t.Fatal("secondary update was not started asynchronously")
}
close(secondary.release)
if text := registry.Render(); !strings.Contains(text, `vehicle_realtime_async_queue_total{protocol="GB32960",status="queued",store="mysql"} 1`) {
t.Fatalf("async queue metric missing:\n%s", text)
}
}
func TestStoreMetricUpdaterRecordsStoreUpdateResults(t *testing.T) {
registry := metrics.NewRegistry()
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001"}
updater := storeMetricUpdater{
store: "redis",
delegate: &contextCheckingRealtimeUpdater{},
registry: registry,
}
if err := updater.Update(context.Background(), env); err != nil {
t.Fatalf("Update() error = %v", err)
}
if text := registry.Render(); !strings.Contains(text, `vehicle_realtime_store_updates_total{protocol="JT808",status="ok",store="redis"} 1`) {
t.Fatalf("store update metric missing:\n%s", text)
}
}
func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) {
@@ -174,6 +195,18 @@ func TestRealtimeAPIExposesPostRawFrameQueryRoute(t *testing.T) {
}
}
func TestRealtimeAPIExposesOperationalRealtimeRoutes(t *testing.T) {
source, err := os.ReadFile("main.go")
if err != nil {
t.Fatalf("read main.go: %v", err)
}
for _, want := range []string{`"/api/realtime/online"`, `"/api/debug/pipeline"`} {
if !strings.Contains(string(source), want) {
t.Fatalf("realtime api should expose operational route %s", want)
}
}
}
type contextCheckingRealtimeUpdater struct {
ctxErr error
count int