feat: build vehicle data platform and production pipeline

go/vehicle-gateway/internal/metrics/metrics.go

@@ -4,12 +4,16 @@ import (
 	"fmt"
 	"math"
 	"net/http"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 )
 
+var processStartedAt = time.Now()
+
 type Labels map[string]string
 
 type Registry struct {
@@ -33,6 +37,56 @@ func NewRegistry() *Registry {
 	}
 }
 
+func RegisterServiceInfo(registry *Registry, service string) {
+	if registry == nil {
+		return
+	}
+	service = strings.TrimSpace(service)
+	if service == "" {
+		service = "vehicle-service"
+	}
+	registry.SetGauge("vehicle_service_info", Labels{"service": service}, 1)
+	RecordProcessRuntime(registry)
+}
+
+func RecordProcessRuntime(registry *Registry) {
+	if registry == nil {
+		return
+	}
+	var mem runtime.MemStats
+	runtime.ReadMemStats(&mem)
+	registry.SetGauge("vehicle_process_start_time_unix_seconds", nil, float64(processStartedAt.Unix()))
+	registry.SetGauge("vehicle_process_uptime_seconds", nil, time.Since(processStartedAt).Seconds())
+	registry.SetGauge("vehicle_process_heap_alloc_bytes", nil, float64(mem.HeapAlloc))
+	registry.SetGauge("vehicle_process_heap_sys_bytes", nil, float64(mem.HeapSys))
+	registry.SetGauge("vehicle_process_goroutines", nil, float64(runtime.NumGoroutine()))
+}
+
+func RegisterKafkaConsumerInfo(registry *Registry, service string, group string, topics []string) {
+	if registry == nil {
+		return
+	}
+	service = strings.TrimSpace(service)
+	if service == "" {
+		service = "vehicle-service"
+	}
+	group = strings.TrimSpace(group)
+	if group == "" {
+		return
+	}
+	for _, topic := range topics {
+		topic = strings.TrimSpace(topic)
+		if topic == "" {
+			continue
+		}
+		registry.SetGauge("vehicle_kafka_consumer_info", Labels{
+			"service": service,
+			"group":   group,
+			"topic":   topic,
+		}, 1)
+	}
+}
+
 func (r *Registry) IncCounter(name string, labels Labels) {
 	r.AddCounter(name, labels, 1)
 }
@@ -90,6 +144,13 @@ func (r *Registry) SetKafkaLag(name string, topic string, partition int, offset
 	}, float64(lag))
 }
 
+func RecordLastActivity(registry *Registry, name string, labels Labels) {
+	if registry == nil {
+		return
+	}
+	registry.SetGauge(name, labels, float64(time.Now().Unix()))
+}
+
 func (r *Registry) ObserveHistogram(name string, labels Labels, buckets []float64, value float64) {
 	name = strings.TrimSpace(name)
 	if r == nil || name == "" {
@@ -265,6 +326,7 @@ func NewHandler(registry *Registry) http.Handler {
 			http.NotFound(w, r)
 			return
 		}
+		RecordProcessRuntime(registry)
 		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
 		_, _ = w.Write([]byte(registry.Render()))
 	})

go/vehicle-gateway/internal/metrics/metrics_test.go

@@ -85,6 +85,65 @@ func TestRegistryRecordsKafkaLagGauge(t *testing.T) {
 	}
 }
 
+func TestRecordLastActivityRendersUnixGauge(t *testing.T) {
+	registry := NewRegistry()
+
+	RecordLastActivity(registry, "vehicle_stat_last_message_unix_seconds", Labels{"topic": "vehicle.fields.go.jt808.v1", "status": "received"})
+
+	text := registry.Render()
+	if !strings.Contains(text, `# TYPE vehicle_stat_last_message_unix_seconds gauge`) ||
+		!strings.Contains(text, `vehicle_stat_last_message_unix_seconds{status="received",topic="vehicle.fields.go.jt808.v1"} `) {
+		t.Fatalf("last activity gauge missing:\n%s", text)
+	}
+}
+
+func TestRegisterServiceInfoRendersStableGauge(t *testing.T) {
+	registry := NewRegistry()
+
+	RegisterServiceInfo(registry, "vehicle-realtime-api")
+
+	text := registry.Render()
+	if !strings.Contains(text, `vehicle_service_info{service="vehicle-realtime-api"} 1`) {
+		t.Fatalf("service info metric missing:\n%s", text)
+	}
+	for _, want := range []string{
+		`# TYPE vehicle_process_start_time_unix_seconds gauge`,
+		`# TYPE vehicle_process_uptime_seconds gauge`,
+		`# TYPE vehicle_process_heap_alloc_bytes gauge`,
+		`# TYPE vehicle_process_heap_sys_bytes gauge`,
+		`# TYPE vehicle_process_goroutines gauge`,
+		`vehicle_process_start_time_unix_seconds `,
+		`vehicle_process_uptime_seconds `,
+		`vehicle_process_heap_alloc_bytes `,
+		`vehicle_process_heap_sys_bytes `,
+		`vehicle_process_goroutines `,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("process runtime metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
+func TestRegisterKafkaConsumerInfoRendersTopics(t *testing.T) {
+	registry := NewRegistry()
+
+	RegisterKafkaConsumerInfo(registry, "vehicle-stat-writer", "go-stat-writer", []string{
+		"vehicle.fields.go.gb32960.v1",
+		"vehicle.fields.go.jt808.v1",
+		"",
+	})
+
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_kafka_consumer_info{group="go-stat-writer",service="vehicle-stat-writer",topic="vehicle.fields.go.gb32960.v1"} 1`,
+		`vehicle_kafka_consumer_info{group="go-stat-writer",service="vehicle-stat-writer",topic="vehicle.fields.go.jt808.v1"} 1`,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("consumer info metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
 func TestHandlerServesPrometheusText(t *testing.T) {
 	registry := NewRegistry()
 	registry.IncCounter("vehicle_kafka_commits_total", Labels{"service": "history"})

go/vehicle-gateway/internal/metrics/pending.go

@@ -0,0 +1,35 @@
+package metrics
+
+import "sync"
+
+// PendingPairGauge keeps process-wide in-flight totals correct when a service
+// has multiple workers updating the same pair of pending gauges.
+type PendingPairGauge struct {
+	mu     sync.Mutex
+	first  int
+	second int
+}
+
+func (g *PendingPairGauge) Add(registry *Registry, firstMetric string, secondMetric string, firstDelta int, secondDelta int) {
+	if g == nil {
+		return
+	}
+	g.mu.Lock()
+	g.first += firstDelta
+	g.second += secondDelta
+	if g.first < 0 {
+		g.first = 0
+	}
+	if g.second < 0 {
+		g.second = 0
+	}
+	first := g.first
+	second := g.second
+	g.mu.Unlock()
+
+	if registry == nil {
+		return
+	}
+	registry.SetGauge(firstMetric, nil, float64(first))
+	registry.SetGauge(secondMetric, nil, float64(second))
+}

go/vehicle-gateway/internal/metrics/recent_latency.go

@@ -0,0 +1,79 @@
+package metrics
+
+import (
+	"math"
+	"sort"
+	"strings"
+	"sync"
+)
+
+// RecentLatencyByKey tracks a bounded in-memory latency window per logical key.
+// It is intentionally small and process-local; Prometheus histograms keep the
+// lifetime view, while this gives capacity-check a recent-window signal.
+type RecentLatencyByKey struct {
+	mu      sync.Mutex
+	size    int
+	windows map[string]*recentLatencyWindow
+}
+
+type recentLatencyWindow struct {
+	values []float64
+	next   int
+	count  int
+}
+
+func NewRecentLatencyByKey(size int) *RecentLatencyByKey {
+	if size <= 0 {
+		size = 1
+	}
+	return &RecentLatencyByKey{
+		size:    size,
+		windows: map[string]*recentLatencyWindow{},
+	}
+}
+
+func (r *RecentLatencyByKey) Observe(key string, value float64) (p99 float64, samples int) {
+	if r == nil {
+		return 0, 0
+	}
+	key = strings.TrimSpace(key)
+	if key == "" {
+		key = "unknown"
+	}
+	if value < 0 || math.IsNaN(value) {
+		value = 0
+	}
+	if math.IsInf(value, 1) {
+		value = math.MaxFloat64
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	window := r.windows[key]
+	if window == nil {
+		window = &recentLatencyWindow{values: make([]float64, r.size)}
+		r.windows[key] = window
+	}
+	window.values[window.next] = value
+	window.next = (window.next + 1) % len(window.values)
+	if window.count < len(window.values) {
+		window.count++
+	}
+	return window.quantileLocked(0.99), window.count
+}
+
+func (w *recentLatencyWindow) quantileLocked(q float64) float64 {
+	if w == nil || w.count == 0 {
+		return 0
+	}
+	snapshot := append([]float64(nil), w.values[:w.count]...)
+	sort.Float64s(snapshot)
+	index := int(math.Ceil(q*float64(len(snapshot)))) - 1
+	if index < 0 {
+		index = 0
+	}
+	if index >= len(snapshot) {
+		index = len(snapshot) - 1
+	}
+	return snapshot[index]
+}

go/vehicle-gateway/internal/metrics/recent_latency_test.go

@@ -0,0 +1,34 @@
+package metrics
+
+import "testing"
+
+func TestRecentLatencyByKeyTracksBoundedP99PerKey(t *testing.T) {
+	tracker := NewRecentLatencyByKey(3)
+
+	p99, samples := tracker.Observe("a", 10)
+	if p99 != 10 || samples != 1 {
+		t.Fatalf("first p99=%v samples=%d", p99, samples)
+	}
+	tracker.Observe("a", 20)
+	p99, samples = tracker.Observe("a", 30)
+	if p99 != 30 || samples != 3 {
+		t.Fatalf("filled p99=%v samples=%d", p99, samples)
+	}
+	p99, samples = tracker.Observe("a", 5)
+	if p99 != 30 || samples != 3 {
+		t.Fatalf("bounded p99=%v samples=%d", p99, samples)
+	}
+	p99, samples = tracker.Observe("b", 7)
+	if p99 != 7 || samples != 1 {
+		t.Fatalf("second key p99=%v samples=%d", p99, samples)
+	}
+}
+
+func TestRecentLatencyByKeySanitizesBadValues(t *testing.T) {
+	tracker := NewRecentLatencyByKey(2)
+
+	p99, samples := tracker.Observe("", -10)
+	if p99 != 0 || samples != 1 {
+		t.Fatalf("p99=%v samples=%d", p99, samples)
+	}
+}