feat: build vehicle data platform and production pipeline

go/vehicle-gateway/internal/eventbus/durable_sink_test.go

@@ -8,8 +8,10 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
+	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
 )
 
 func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
@@ -167,6 +169,156 @@ func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) {
 	}
 }
 
+func TestDurableSinkRecordsSpoolAndReplayMetrics(t *testing.T) {
+	dir := t.TempDir()
+	registry := metrics.NewRegistry()
+	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
+	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
+	env := durableTestEnvelope()
+
+	if err := sink.PublishRaw(context.Background(), env); err != nil {
+		t.Fatalf("PublishRaw() error = %v", err)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_durable_spool_records_total{kind="raw",name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_backlog_files{name="nats"} 1`,
+		`vehicle_durable_spool_oldest_age_seconds{name="nats"}`,
+	} {
+		if !containsString(text, want) {
+			t.Fatalf("spool metric missing %s:\n%s", want, text)
+		}
+	}
+
+	delegate.rawErrors = nil
+	if err := sink.ReplayOnce(context.Background()); err != nil {
+		t.Fatalf("ReplayOnce() error = %v", err)
+	}
+	text = registry.Render()
+	for _, want := range []string{
+		`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_replay_records_total{kind="all",name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
+		`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
+	} {
+		if !containsString(text, want) {
+			t.Fatalf("replay metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
+func TestDurableSinkInitializesBacklogMetrics(t *testing.T) {
+	dir := t.TempDir()
+	registry := metrics.NewRegistry()
+	_ = NewDurableSink(&scriptedSink{}, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
+
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
+		`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
+	} {
+		if !containsString(text, want) {
+			t.Fatalf("initial spool metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
+func TestDurableSinkRecordsReplayPublishErrorMetrics(t *testing.T) {
+	dir := t.TempDir()
+	registry := metrics.NewRegistry()
+	env := durableTestEnvelope()
+	writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
+	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
+	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "kafka"})
+
+	if err := sink.ReplayOnce(context.Background()); err == nil {
+		t.Fatal("ReplayOnce() error = nil, want delegate publish error")
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_durable_spool_replay_total{name="kafka",status="publish_error"} 1`,
+		`vehicle_durable_spool_replay_records_total{kind="all",name="kafka",status="publish_error"} 1`,
+		`vehicle_durable_spool_replay_records_total{kind="raw",name="kafka",status="publish_error"} 1`,
+		`vehicle_durable_spool_backlog_files{name="kafka"} 1`,
+	} {
+		if !containsString(text, want) {
+			t.Fatalf("replay error metric missing %s:\n%s", want, text)
+		}
+	}
+	if files := spoolFiles(t, dir); len(files) != 1 {
+		t.Fatalf("failed replay should keep spool file, files=%#v", files)
+	}
+}
+
+func TestDurableBacklogStatsCountsAllFilesAndOldestAge(t *testing.T) {
+	dir := t.TempDir()
+	env := durableTestEnvelope()
+	now := time.Date(2026, 7, 12, 15, 0, 0, 0, time.UTC)
+	oldFile := filepath.Join(dir, "0001-raw.json")
+	newFile := filepath.Join(dir, "0002-fields.json")
+	writeDurableRecord(t, oldFile, durableRecord{Kind: "raw", Envelope: env})
+	writeDurableRecord(t, newFile, durableRecord{Kind: "fields", Envelope: env})
+	if err := os.Chtimes(oldFile, now.Add(-10*time.Minute), now.Add(-10*time.Minute)); err != nil {
+		t.Fatalf("chtimes old file: %v", err)
+	}
+	if err := os.Chtimes(newFile, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil {
+		t.Fatalf("chtimes new file: %v", err)
+	}
+
+	count, oldestAge, err := durableBacklogStats(dir, now)
+	if err != nil {
+		t.Fatalf("durableBacklogStats() error = %v", err)
+	}
+	if count != 2 {
+		t.Fatalf("count = %d, want 2", count)
+	}
+	if oldestAge != 10*time.Minute {
+		t.Fatalf("oldestAge = %s, want 10m", oldestAge)
+	}
+}
+
+func TestDurableSinkQuarantinesBadRecordAndReplaysRemainingFiles(t *testing.T) {
+	dir := t.TempDir()
+	registry := metrics.NewRegistry()
+	env := durableTestEnvelope()
+	if err := os.WriteFile(filepath.Join(dir, "0001-bad.json"), []byte("{bad json"), 0o640); err != nil {
+		t.Fatalf("write bad durable record: %v", err)
+	}
+	writeDurableRecord(t, filepath.Join(dir, "0002-raw.json"), durableRecord{Kind: "raw", Envelope: env})
+	delegate := &scriptedSink{}
+	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
+
+	if err := sink.ReplayOnce(context.Background()); err != nil {
+		t.Fatalf("ReplayOnce() error = %v", err)
+	}
+	if delegate.rawCalls != 1 {
+		t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
+	}
+	if files := spoolFiles(t, dir); len(files) != 0 {
+		t.Fatalf("normal spool files after replay = %#v, want none", files)
+	}
+	badFiles, err := filepath.Glob(filepath.Join(dir, "*.bad"))
+	if err != nil {
+		t.Fatalf("glob bad files: %v", err)
+	}
+	if len(badFiles) != 1 {
+		t.Fatalf("bad files = %#v, want one quarantined file", badFiles)
+	}
+	text := registry.Render()
+	for _, want := range []string{
+		`vehicle_durable_spool_replay_total{name="nats",status="read_error"} 1`,
+		`vehicle_durable_spool_replay_total{name="nats",status="quarantined"} 1`,
+		`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
+		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
+	} {
+		if !containsString(text, want) {
+			t.Fatalf("quarantine metric missing %s:\n%s", want, text)
+		}
+	}
+}
+
 func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
 	reader := &fakeNameReader{
 		batches: [][]string{