lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus/durable_sink_test.go

package eventbus

import (
	"context"
	"encoding/json"
	"errors"
	"io"
	"os"
	"path/filepath"
	"testing"
	"time"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)

func TestDurableSinkSpoolsUnifiedWhenRawWasSpooled(t *testing.T) {
	dir := t.TempDir()
	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
	env := durableTestEnvelope()

	if err := sink.PublishRaw(context.Background(), env); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	if err := sink.PublishUnified(context.Background(), env); err != nil {
		t.Fatalf("PublishUnified() error = %v", err)
	}

	if delegate.rawCalls != 1 {
		t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
	}
	if delegate.unifiedCalls != 0 {
		t.Fatalf("unified should not be delegated while raw is spooled, calls = %d", delegate.unifiedCalls)
	}
	files := spoolFiles(t, dir)
	if len(files) != 2 {
		t.Fatalf("spool files = %d, want 2: %#v", len(files), files)
	}
	assertSpoolKind(t, files[0], "raw")
	assertSpoolKind(t, files[1], "unified")
}

func TestDurableSinkReplayPublishesInFileOrderAndDeletesFiles(t *testing.T) {
	dir := t.TempDir()
	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
	env := durableTestEnvelope()

	if err := sink.PublishRaw(context.Background(), env); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	if err := sink.PublishUnified(context.Background(), env); err != nil {
		t.Fatalf("PublishUnified() error = %v", err)
	}
	delegate.rawErrors = nil

	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}

	got := delegate.calls
	want := []string{"raw", "raw", "unified"}
	if len(got) != len(want) {
		t.Fatalf("calls = %#v, want %#v", got, want)
	}
	for i := range want {
		if got[i] != want[i] {
			t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want[i], got)
		}
	}
	if files := spoolFiles(t, dir); len(files) != 0 {
		t.Fatalf("spool files after replay = %#v, want none", files)
	}
}

func TestDurableSinkReplayPublishesRawBeforeUnifiedEvenWhenFilesAreOutOfOrder(t *testing.T) {
	dir := t.TempDir()
	env := durableTestEnvelope()
	writeDurableRecord(t, filepath.Join(dir, "0001-unified.json"), durableRecord{Kind: "unified", Envelope: env})
	writeDurableRecord(t, filepath.Join(dir, "0002-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	delegate := &scriptedSink{}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir})

	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}

	got := delegate.calls
	want := []string{"raw", "unified"}
	if len(got) != len(want) {
		t.Fatalf("calls = %#v, want %#v", got, want)
	}
	for i := range want {
		if got[i] != want[i] {
			t.Fatalf("calls[%d] = %q, want %q; all=%#v", i, got[i], want, got)
		}
	}
}

func TestDurableSinkReplayLoopTickUsesUncancelledContext(t *testing.T) {
	dir := t.TempDir()
	env := durableTestEnvelope()
	writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	delegate := &contextCheckingReplaySink{}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir})
	parent, cancel := context.WithCancel(context.Background())
	cancel()

	if err := sink.replayOnceFromLoop(parent); err != nil {
		t.Fatalf("replayOnceFromLoop() error = %v", err)
	}
	if delegate.rawCtxErr != nil {
		t.Fatalf("delegate saw cancelled context: %v", delegate.rawCtxErr)
	}
	if delegate.rawCalls != 1 {
		t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
	}
	if files := spoolFiles(t, dir); len(files) != 0 {
		t.Fatalf("spool files after replay = %#v, want none", files)
	}
}

func TestDurableSinkReplayLoopTickRespectsBatchSize(t *testing.T) {
	dir := t.TempDir()
	env := durableTestEnvelope()
	for index := 0; index < 5; index++ {
		writeDurableRecord(t, filepath.Join(dir, "000"+string(rune('1'+index))+"-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	}
	delegate := &contextCheckingReplaySink{}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, ReplayBatchSize: 2})

	if err := sink.replayOnceFromLoop(context.Background()); err != nil {
		t.Fatalf("replayOnceFromLoop() error = %v", err)
	}
	if delegate.rawCalls != 2 {
		t.Fatalf("raw calls = %d, want 2", delegate.rawCalls)
	}
	if files := spoolFiles(t, dir); len(files) != 3 {
		t.Fatalf("spool files after replay = %d, want 3: %#v", len(files), files)
	}
}

func TestDurableSinkReplayUsesBatchPublisherWhenAvailable(t *testing.T) {
	dir := t.TempDir()
	env := durableTestEnvelope()
	writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	writeDurableRecord(t, filepath.Join(dir, "0002-unified.json"), durableRecord{Kind: "unified", Envelope: env})
	delegate := &batchRecordingSink{}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir})

	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	if delegate.batchCalls != 1 {
		t.Fatalf("batch calls = %d, want 1", delegate.batchCalls)
	}
	if delegate.rawCalls != 0 || delegate.unifiedCalls != 0 {
		t.Fatalf("individual publish should not be used, raw=%d unified=%d", delegate.rawCalls, delegate.unifiedCalls)
	}
	if got, want := len(delegate.records), 2; got != want {
		t.Fatalf("batch record count = %d, want %d", got, want)
	}
	if delegate.records[0].Kind != "raw" || delegate.records[1].Kind != "unified" {
		t.Fatalf("batch order = %#v", delegate.records)
	}
	if files := spoolFiles(t, dir); len(files) != 0 {
		t.Fatalf("spool files after replay = %#v, want none", files)
	}
}

func TestDurableSinkRecordsSpoolAndReplayMetrics(t *testing.T) {
	dir := t.TempDir()
	registry := metrics.NewRegistry()
	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})
	env := durableTestEnvelope()

	if err := sink.PublishRaw(context.Background(), env); err != nil {
		t.Fatalf("PublishRaw() error = %v", err)
	}
	text := registry.Render()
	for _, want := range []string{
		`vehicle_durable_spool_records_total{kind="raw",name="nats",status="ok"} 1`,
		`vehicle_durable_spool_backlog_files{name="nats"} 1`,
		`vehicle_durable_spool_oldest_age_seconds{name="nats"}`,
	} {
		if !containsString(text, want) {
			t.Fatalf("spool metric missing %s:\n%s", want, text)
		}
	}

	delegate.rawErrors = nil
	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	text = registry.Render()
	for _, want := range []string{
		`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
		`vehicle_durable_spool_replay_records_total{kind="all",name="nats",status="ok"} 1`,
		`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
		`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
	} {
		if !containsString(text, want) {
			t.Fatalf("replay metric missing %s:\n%s", want, text)
		}
	}
}

func TestDurableSinkInitializesBacklogMetrics(t *testing.T) {
	dir := t.TempDir()
	registry := metrics.NewRegistry()
	_ = NewDurableSink(&scriptedSink{}, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})

	text := registry.Render()
	for _, want := range []string{
		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
		`vehicle_durable_spool_oldest_age_seconds{name="nats"} 0`,
	} {
		if !containsString(text, want) {
			t.Fatalf("initial spool metric missing %s:\n%s", want, text)
		}
	}
}

func TestDurableSinkRecordsReplayPublishErrorMetrics(t *testing.T) {
	dir := t.TempDir()
	registry := metrics.NewRegistry()
	env := durableTestEnvelope()
	writeDurableRecord(t, filepath.Join(dir, "0001-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	delegate := &scriptedSink{rawErrors: []error{errSpoolTest}}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "kafka"})

	if err := sink.ReplayOnce(context.Background()); err == nil {
		t.Fatal("ReplayOnce() error = nil, want delegate publish error")
	}
	text := registry.Render()
	for _, want := range []string{
		`vehicle_durable_spool_replay_total{name="kafka",status="publish_error"} 1`,
		`vehicle_durable_spool_replay_records_total{kind="all",name="kafka",status="publish_error"} 1`,
		`vehicle_durable_spool_replay_records_total{kind="raw",name="kafka",status="publish_error"} 1`,
		`vehicle_durable_spool_backlog_files{name="kafka"} 1`,
	} {
		if !containsString(text, want) {
			t.Fatalf("replay error metric missing %s:\n%s", want, text)
		}
	}
	if files := spoolFiles(t, dir); len(files) != 1 {
		t.Fatalf("failed replay should keep spool file, files=%#v", files)
	}
}

func TestDurableBacklogStatsCountsAllFilesAndOldestAge(t *testing.T) {
	dir := t.TempDir()
	env := durableTestEnvelope()
	now := time.Date(2026, 7, 12, 15, 0, 0, 0, time.UTC)
	oldFile := filepath.Join(dir, "0001-raw.json")
	newFile := filepath.Join(dir, "0002-fields.json")
	writeDurableRecord(t, oldFile, durableRecord{Kind: "raw", Envelope: env})
	writeDurableRecord(t, newFile, durableRecord{Kind: "fields", Envelope: env})
	if err := os.Chtimes(oldFile, now.Add(-10*time.Minute), now.Add(-10*time.Minute)); err != nil {
		t.Fatalf("chtimes old file: %v", err)
	}
	if err := os.Chtimes(newFile, now.Add(-30*time.Second), now.Add(-30*time.Second)); err != nil {
		t.Fatalf("chtimes new file: %v", err)
	}

	count, oldestAge, err := durableBacklogStats(dir, now)
	if err != nil {
		t.Fatalf("durableBacklogStats() error = %v", err)
	}
	if count != 2 {
		t.Fatalf("count = %d, want 2", count)
	}
	if oldestAge != 10*time.Minute {
		t.Fatalf("oldestAge = %s, want 10m", oldestAge)
	}
}

func TestDurableSinkQuarantinesBadRecordAndReplaysRemainingFiles(t *testing.T) {
	dir := t.TempDir()
	registry := metrics.NewRegistry()
	env := durableTestEnvelope()
	if err := os.WriteFile(filepath.Join(dir, "0001-bad.json"), []byte("{bad json"), 0o640); err != nil {
		t.Fatalf("write bad durable record: %v", err)
	}
	writeDurableRecord(t, filepath.Join(dir, "0002-raw.json"), durableRecord{Kind: "raw", Envelope: env})
	delegate := &scriptedSink{}
	sink := NewDurableSink(delegate, DurableConfig{Directory: dir, Metrics: registry, Name: "nats"})

	if err := sink.ReplayOnce(context.Background()); err != nil {
		t.Fatalf("ReplayOnce() error = %v", err)
	}
	if delegate.rawCalls != 1 {
		t.Fatalf("raw calls = %d, want 1", delegate.rawCalls)
	}
	if files := spoolFiles(t, dir); len(files) != 0 {
		t.Fatalf("normal spool files after replay = %#v, want none", files)
	}
	badFiles, err := filepath.Glob(filepath.Join(dir, "*.bad"))
	if err != nil {
		t.Fatalf("glob bad files: %v", err)
	}
	if len(badFiles) != 1 {
		t.Fatalf("bad files = %#v, want one quarantined file", badFiles)
	}
	text := registry.Render()
	for _, want := range []string{
		`vehicle_durable_spool_replay_total{name="nats",status="read_error"} 1`,
		`vehicle_durable_spool_replay_total{name="nats",status="quarantined"} 1`,
		`vehicle_durable_spool_replay_total{name="nats",status="ok"} 1`,
		`vehicle_durable_spool_replay_records_total{kind="raw",name="nats",status="ok"} 1`,
		`vehicle_durable_spool_backlog_files{name="nats"} 0`,
	} {
		if !containsString(text, want) {
			t.Fatalf("quarantine metric missing %s:\n%s", want, text)
		}
	}
}

func TestDurableFilesFromReaderStopsAfterLimitedJSONBatch(t *testing.T) {
	reader := &fakeNameReader{
		batches: [][]string{
			{"0001-raw.json", "notes.txt", "0002-unified.json"},
			{"0003-raw.json"},
		},
	}

	files, err := durableFilesFromReader("/spool", 2, reader)
	if err != nil {
		t.Fatalf("durableFilesFromReader() error = %v", err)
	}
	if reader.calls != 1 {
		t.Fatalf("Readdirnames calls = %d, want 1", reader.calls)
	}
	want := []string{filepath.Join("/spool", "0001-raw.json"), filepath.Join("/spool", "0002-unified.json")}
	if len(files) != len(want) {
		t.Fatalf("files = %#v, want %#v", files, want)
	}
	for i := range want {
		if files[i] != want[i] {
			t.Fatalf("files[%d] = %q, want %q; all=%#v", i, files[i], want[i], files)
		}
	}
}

func durableTestEnvelope() envelope.FrameEnvelope {
	return envelope.FrameEnvelope{
		Protocol:     envelope.ProtocolJT808,
		MessageID:    "0x0200",
		Phone:        "013079963379",
		VIN:          "LKLG7C4E3NA774736",
		Sequence:     183,
		EventTimeMS:  1782903940000,
		ReceivedAtMS: 1782917751903,
		RawHex:       "0200002201307996337900B7",
	}
}

type fakeNameReader struct {
	batches [][]string
	calls   int
}

func (r *fakeNameReader) Readdirnames(int) ([]string, error) {
	r.calls++
	if len(r.batches) == 0 {
		return nil, io.EOF
	}
	batch := r.batches[0]
	r.batches = r.batches[1:]
	if len(r.batches) == 0 {
		return batch, io.EOF
	}
	return batch, nil
}

func writeDurableRecord(t *testing.T, path string, record durableRecord) {
	t.Helper()
	payload, err := json.Marshal(record)
	if err != nil {
		t.Fatalf("marshal durable record: %v", err)
	}
	if err := os.WriteFile(path, payload, 0o640); err != nil {
		t.Fatalf("write durable record: %v", err)
	}
}

func spoolFiles(t *testing.T, dir string) []string {
	t.Helper()
	matches, err := filepath.Glob(filepath.Join(dir, "*.json"))
	if err != nil {
		t.Fatalf("glob spool files: %v", err)
	}
	return matches
}

func assertSpoolKind(t *testing.T, path string, kind string) {
	t.Helper()
	payload, err := os.ReadFile(path)
	if err != nil {
		t.Fatalf("read spool file: %v", err)
	}
	if !containsString(string(payload), `"kind":"`+kind+`"`) {
		t.Fatalf("spool file %s payload = %s, want kind %q", path, payload, kind)
	}
}

func containsString(value string, needle string) bool {
	return len(needle) == 0 || (len(value) >= len(needle) && indexString(value, needle) >= 0)
}

func indexString(value string, needle string) int {
	for i := 0; i+len(needle) <= len(value); i++ {
		if value[i:i+len(needle)] == needle {
			return i
		}
	}
	return -1
}

var errSpoolTest = errors.New("delegate unavailable")

type scriptedSink struct {
	rawErrors     []error
	unifiedErrors []error
	rawCalls      int
	unifiedCalls  int
	calls         []string
}

func (s *scriptedSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
	s.rawCalls++
	s.calls = append(s.calls, "raw")
	if len(s.rawErrors) > 0 {
		err := s.rawErrors[0]
		s.rawErrors = s.rawErrors[1:]
		return err
	}
	return nil
}

func (s *scriptedSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
	s.unifiedCalls++
	s.calls = append(s.calls, "unified")
	if len(s.unifiedErrors) > 0 {
		err := s.unifiedErrors[0]
		s.unifiedErrors = s.unifiedErrors[1:]
		return err
	}
	return nil
}

func (s *scriptedSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
	s.calls = append(s.calls, "fields")
	return nil
}

func (s *scriptedSink) Close() error {
	return nil
}

type contextCheckingReplaySink struct {
	rawCtxErr     error
	unifiedCtxErr error
	rawCalls      int
	unifiedCalls  int
}

func (s *contextCheckingReplaySink) PublishRaw(ctx context.Context, _ envelope.FrameEnvelope) error {
	s.rawCtxErr = ctx.Err()
	s.rawCalls++
	return s.rawCtxErr
}

func (s *contextCheckingReplaySink) PublishUnified(ctx context.Context, _ envelope.FrameEnvelope) error {
	s.unifiedCtxErr = ctx.Err()
	s.unifiedCalls++
	return s.unifiedCtxErr
}

func (s *contextCheckingReplaySink) PublishFields(ctx context.Context, _ envelope.FrameEnvelope) error {
	return ctx.Err()
}

func (s *contextCheckingReplaySink) Close() error {
	return nil
}

type batchRecordingSink struct {
	batchCalls   int
	records      []durableRecord
	rawCalls     int
	unifiedCalls int
}

func (s *batchRecordingSink) PublishRecords(_ context.Context, records []durableRecord) error {
	s.batchCalls++
	s.records = append(s.records, records...)
	return nil
}

func (s *batchRecordingSink) PublishRaw(context.Context, envelope.FrameEnvelope) error {
	s.rawCalls++
	return nil
}

func (s *batchRecordingSink) PublishUnified(context.Context, envelope.FrameEnvelope) error {
	s.unifiedCalls++
	return nil
}

func (s *batchRecordingSink) PublishFields(context.Context, envelope.FrameEnvelope) error {
	return nil
}

func (s *batchRecordingSink) Close() error {
	return nil
}