feat(go): improve realtime pipeline resilience

go/vehicle-gateway/cmd/realtime-api/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"os"
 	"strings"
 	"sync"
@@ -103,7 +104,7 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) {
 	primary := &contextCheckingRealtimeUpdater{}
 	secondary := &blockingRealtimeUpdater{started: make(chan struct{}), release: make(chan struct{})}
 	registry := metrics.NewRegistry()
-	updater := newAsyncSecondaryRealtimeUpdater(primary, secondary, 8, 1, registry)
+	updater := newAsyncSecondaryRealtimeUpdater(primary, secondary, 8, 1, registry, nil)
 	defer updater.Close()
 
 	start := time.Now()
@@ -122,8 +123,14 @@ func TestAsyncSecondaryRealtimeUpdaterDoesNotBlockPrimaryPath(t *testing.T) {
 		t.Fatal("secondary update was not started asynchronously")
 	}
 	close(secondary.release)
-	if text := registry.Render(); !strings.Contains(text, `vehicle_realtime_async_queue_total{protocol="GB32960",status="queued",store="mysql"} 1`) {
-		t.Fatalf("async queue metric missing:\n%s", text)
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_realtime_async_queue_total{protocol="GB32960",status="queued",store="mysql"} 1`,
+		`vehicle_realtime_async_queue_depth{protocol="GB32960",store="mysql"}`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("async queue metric missing %s:\n%s", want, text)
+		}
 	}
 }
 
@@ -139,11 +146,49 @@ func TestStoreMetricUpdaterRecordsStoreUpdateResults(t *testing.T) {
 	if err := updater.Update(context.Background(), env); err != nil {
 		t.Fatalf("Update() error = %v", err)
 	}
-	if text := registry.Render(); !strings.Contains(text, `vehicle_realtime_store_updates_total{protocol="JT808",status="ok",store="redis"} 1`) {
-		t.Fatalf("store update metric missing:\n%s", text)
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_realtime_store_updates_total{protocol="JT808",status="ok",store="redis"} 1`,
+		`vehicle_realtime_store_update_duration_ms{protocol="JT808",status="ok",store="redis"}`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("store update metric missing %s:\n%s", want, text)
+		}
 	}
 }
 
+func TestRetryRealtimeUpdaterRetriesTransientMySQLDeadlock(t *testing.T) {
+	delegate := &retryOnceRealtimeUpdater{err: errors.New("Error 1213 (40001): Deadlock found when trying to get lock")}
+	updater := retryRealtimeUpdater{delegate: delegate, attempts: 3}
+
+	if err := updater.Update(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001"}); err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+	if delegate.count != 2 {
+		t.Fatalf("delegate updates = %d, want 2", delegate.count)
+	}
+}
+
+func TestAsyncSecondaryRealtimeUpdaterLogsSecondaryErrors(t *testing.T) {
+	env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", EventID: "event-1"}
+	logger := &recordingRealtimeLogger{}
+	updater := newAsyncSecondaryRealtimeUpdater(nil, failingRealtimeUpdater{}, 8, 1, nil, logger)
+	defer updater.Close()
+
+	if err := updater.Update(context.Background(), env); err != nil {
+		t.Fatalf("Update() error = %v", err)
+	}
+
+	deadline := time.Now().Add(time.Second)
+	for time.Now().Before(deadline) {
+		if logger.containsError("async realtime secondary update failed") {
+			return
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	t.Fatalf("secondary error was not logged: %#v", logger.errors)
+}
+
 func TestKafkaTopicsFromEnvDefaultsToGoRawTopics(t *testing.T) {
 	got := strings.Join(kafkaTopicsFromEnv(), ",")
 	want := "vehicle.raw.go.gb32960.v1,vehicle.raw.go.jt808.v1,vehicle.raw.go.yutong-mqtt.v1"
@@ -235,6 +280,30 @@ func (discardRealtimeLogger) Info(string, ...any)  {}
 func (discardRealtimeLogger) Error(string, ...any) {}
 func (discardRealtimeLogger) Warn(string, ...any)  {}
 
+type recordingRealtimeLogger struct {
+	mu     sync.Mutex
+	errors []string
+}
+
+func (l *recordingRealtimeLogger) Info(string, ...any) {}
+func (l *recordingRealtimeLogger) Warn(string, ...any) {}
+func (l *recordingRealtimeLogger) Error(message string, _ ...any) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.errors = append(l.errors, message)
+}
+
+func (l *recordingRealtimeLogger) containsError(message string) bool {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	for _, got := range l.errors {
+		if got == message {
+			return true
+		}
+	}
+	return false
+}
+
 type blockingRealtimeUpdater struct {
 	started chan struct{}
 	release chan struct{}
@@ -246,3 +315,24 @@ func (u *blockingRealtimeUpdater) Update(context.Context, envelope.FrameEnvelope
 	<-u.release
 	return nil
 }
+
+type failingRealtimeUpdater struct{}
+
+func (failingRealtimeUpdater) Update(context.Context, envelope.FrameEnvelope) error {
+	return errTestRealtimeUpdate
+}
+
+var errTestRealtimeUpdate = errors.New("test realtime update failed")
+
+type retryOnceRealtimeUpdater struct {
+	err   error
+	count int
+}
+
+func (u *retryOnceRealtimeUpdater) Update(context.Context, envelope.FrameEnvelope) error {
+	u.count++
+	if u.count == 1 {
+		return u.err
+	}
+	return nil
+}