lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus/durable_sink.go

package eventbus

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
	"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)

type DurableConfig struct {
	Directory       string
	ReplayBatchSize int
	Metrics         *metrics.Registry
	Name            string
}

type DurableSink struct {
	delegate Sink
	dir      string
	metrics  *metrics.Registry
	name     string

	mu              sync.Mutex
	seq             uint64
	rawPending      map[string]struct{}
	replayBatchSize int
}

type durableRecord struct {
	Kind     string                 `json:"kind"`
	Envelope envelope.FrameEnvelope `json:"envelope"`
}

type durableRecordFile struct {
	path   string
	record durableRecord
}

type recordPublishingSink interface {
	PublishRecords(context.Context, []durableRecord) error
}

const durableReplayOperationTimeout = 30 * time.Second

func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
	if delegate == nil {
		panic("durable delegate sink must not be nil")
	}
	s := &DurableSink{
		delegate:        delegate,
		dir:             strings.TrimSpace(cfg.Directory),
		metrics:         cfg.Metrics,
		name:            durableMetricName(cfg.Name),
		rawPending:      map[string]struct{}{},
		replayBatchSize: cfg.ReplayBatchSize,
	}
	s.recordBacklogAfterReplay()
	return s
}

func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
	if err := s.delegate.PublishRaw(ctx, env); err == nil {
		return nil
	}
	if err := s.spool("raw", env); err != nil {
		return err
	}
	s.markRawPending(env)
	return nil
}

func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error {
	if s.isRawPending(env) {
		return s.spool("unified", env)
	}
	if err := s.delegate.PublishUnified(ctx, env); err == nil {
		return nil
	}
	return s.spool("unified", env)
}

func (s *DurableSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error {
	if s.isRawPending(env) {
		return s.spool("fields", env)
	}
	if err := s.delegate.PublishFields(ctx, env); err == nil {
		return nil
	}
	return s.spool("fields", env)
}

func (s *DurableSink) ReplayOnce(ctx context.Context) error {
	return s.replay(ctx, 0)
}

func (s *DurableSink) replay(ctx context.Context, limit int) error {
	files, err := durableFiles(s.dir, limit)
	if err != nil {
		s.recordReplay("list_error", 0)
		return err
	}
	s.recordBacklogAfterReplay()
	records := make([]durableRecordFile, 0, len(files))
	for _, file := range files {
		record, err := readDurableRecord(file)
		if err != nil {
			s.recordReplay("read_error", 1)
			if quarantineErr := quarantineDurableFile(file); quarantineErr != nil {
				s.recordReplay("quarantine_error", 1)
				return fmt.Errorf("quarantine unreadable durable record %s: read error: %w; quarantine error: %v", file, err, quarantineErr)
			}
			s.recordReplay("quarantined", 1)
			continue
		}
		records = append(records, durableRecordFile{path: file, record: record})
	}
	if len(records) == 0 {
		s.recordReplay("empty", 0)
		s.recordBacklogAfterReplay()
		return nil
	}
	sortDurableRecords(records)
	if publisher, ok := s.delegate.(recordPublishingSink); ok {
		durableRecords := make([]durableRecord, 0, len(records))
		for _, item := range records {
			durableRecords = append(durableRecords, item.record)
		}
		if err := publisher.PublishRecords(ctx, durableRecords); err != nil {
			s.recordReplay("publish_error", len(records))
			s.recordReplayRecords(records, "publish_error")
			return err
		}
		for _, item := range records {
			if err := os.Remove(item.path); err != nil {
				s.recordReplay("delete_error", len(records))
				return err
			}
			if item.record.Kind == "raw" {
				s.clearRawPending(item.record.Envelope)
			}
		}
		s.recordReplay("ok", len(records))
		s.recordReplayRecords(records, "ok")
		s.recordBacklogAfterReplay()
		return nil
	}
	for _, item := range records {
		if err := s.publishRecord(ctx, item.record); err != nil {
			s.recordReplay("publish_error", len(records))
			s.recordReplayRecord(item.record, "publish_error")
			return err
		}
		if err := os.Remove(item.path); err != nil {
			s.recordReplay("delete_error", len(records))
			return err
		}
		if item.record.Kind == "raw" {
			s.clearRawPending(item.record.Envelope)
		}
		s.recordReplayRecord(item.record, "ok")
	}
	s.recordReplay("ok", len(records))
	s.recordBacklogAfterReplay()
	return nil
}

func sortDurableRecords(records []durableRecordFile) {
	sort.SliceStable(records, func(i, j int) bool {
		leftEvent := records[i].record.Envelope.StableEventID()
		rightEvent := records[j].record.Envelope.StableEventID()
		if leftEvent == rightEvent {
			return durableKindOrder(records[i].record.Kind) < durableKindOrder(records[j].record.Kind)
		}
		return records[i].path < records[j].path
	})
}

func durableKindOrder(kind string) int {
	if kind == "raw" {
		return 0
	}
	if kind == "unified" {
		return 1
	}
	if kind == "fields" {
		return 2
	}
	return 3
}

func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) {
	if interval <= 0 {
		interval = time.Second
	}
	ticker := time.NewTicker(interval)
	defer ticker.Stop()
	for {
		select {
		case <-ctx.Done():
			return
		case <-ticker.C:
			if err := s.replayOnceFromLoop(ctx); err != nil && onError != nil {
				onError(err)
			}
		}
	}
}

func (s *DurableSink) replayOnceFromLoop(ctx context.Context) error {
	replayCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), durableReplayOperationTimeout)
	defer cancel()
	return s.replay(replayCtx, s.replayBatchSize)
}

func (s *DurableSink) Close() error {
	return s.delegate.Close()
}

func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error {
	switch record.Kind {
	case "raw":
		return s.delegate.PublishRaw(ctx, record.Envelope)
	case "unified":
		return s.delegate.PublishUnified(ctx, record.Envelope)
	case "fields":
		return s.delegate.PublishFields(ctx, record.Envelope)
	default:
		return errUnknownRecordKind(record.Kind)
	}
}

func errUnknownRecordKind(kind string) error {
	return fmt.Errorf("unknown durable record kind %q", kind)
}

func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
	if s.dir == "" {
		return s.spoolError(kind, fmt.Errorf("durable spool directory is empty"))
	}
	if env.EventID == "" {
		env.EventID = env.StableEventID()
	}
	if env.ParseStatus == "" {
		env.ParseStatus = envelope.ParseOK
	}
	if err := os.MkdirAll(s.dir, 0o750); err != nil {
		return s.spoolError(kind, err)
	}
	payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
	if err != nil {
		return s.spoolError(kind, err)
	}
	name := s.nextFileName(env, kind)
	path := filepath.Join(s.dir, name)
	tmp := path + ".tmp"
	if err := os.WriteFile(tmp, payload, 0o640); err != nil {
		return s.spoolError(kind, err)
	}
	if err := os.Rename(tmp, path); err != nil {
		return s.spoolError(kind, err)
	}
	s.recordSpool(kind, "ok")
	s.recordBacklogAfterReplay()
	return nil
}

func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.seq++
	eventID := env.StableEventID()
	if len(eventID) > 12 {
		eventID = eventID[:12]
	}
	return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID)
}

func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.rawPending[env.StableEventID()] = struct{}{}
}

func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) {
	s.mu.Lock()
	defer s.mu.Unlock()
	delete(s.rawPending, env.StableEventID())
}

func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool {
	s.mu.Lock()
	defer s.mu.Unlock()
	_, ok := s.rawPending[env.StableEventID()]
	return ok
}

func readDurableRecord(path string) (durableRecord, error) {
	var record durableRecord
	payload, err := os.ReadFile(path)
	if err != nil {
		return record, err
	}
	return record, json.Unmarshal(payload, &record)
}

func quarantineDurableFile(path string) error {
	target := path + ".bad"
	if _, err := os.Stat(target); err == nil {
		target = fmt.Sprintf("%s.%d.bad", path, time.Now().UnixNano())
	} else if !os.IsNotExist(err) {
		return err
	}
	return os.Rename(path, target)
}

func durableFiles(dir string, limit int) ([]string, error) {
	if limit > 0 {
		handle, err := os.Open(dir)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, nil
			}
			return nil, err
		}
		defer handle.Close()
		return durableFilesFromReader(dir, limit, handle)
	}
	files, err := filepath.Glob(filepath.Join(dir, "*.json"))
	if err != nil {
		return nil, err
	}
	sort.Strings(files)
	return files, nil
}

type durableNameReader interface {
	Readdirnames(int) ([]string, error)
}

func durableFilesFromReader(dir string, limit int, reader durableNameReader) ([]string, error) {
	if limit <= 0 {
		return nil, nil
	}
	files := make([]string, 0, limit)
	for len(files) < limit {
		names, err := reader.Readdirnames(limit - len(files))
		for _, name := range names {
			if !strings.HasSuffix(name, ".json") {
				continue
			}
			files = append(files, filepath.Join(dir, name))
			if len(files) >= limit {
				break
			}
		}
		if err != nil {
			if errorsIsEOF(err) {
				break
			}
			return nil, err
		}
		if len(names) == 0 {
			break
		}
	}
	sort.Strings(files)
	return files, nil
}

func errorsIsEOF(err error) bool {
	return err == io.EOF
}

func durableMetricName(name string) string {
	name = strings.TrimSpace(name)
	if name == "" {
		return "default"
	}
	return name
}

func (s *DurableSink) spoolError(kind string, err error) error {
	s.recordSpool(kind, "error")
	return err
}

func (s *DurableSink) recordSpool(kind string, status string) {
	if s == nil || s.metrics == nil {
		return
	}
	s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{
		"name":   s.name,
		"kind":   kind,
		"status": status,
	})
}

func (s *DurableSink) recordReplay(status string, records int) {
	if s == nil || s.metrics == nil {
		return
	}
	labels := metrics.Labels{"name": s.name, "status": status}
	s.metrics.IncCounter("vehicle_durable_spool_replay_total", labels)
	if records > 0 {
		s.metrics.AddCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
			"name":   s.name,
			"kind":   "all",
			"status": status,
		}, float64(records))
	}
}

func (s *DurableSink) recordReplayRecords(records []durableRecordFile, status string) {
	for _, item := range records {
		s.recordReplayRecord(item.record, status)
	}
}

func (s *DurableSink) recordReplayRecord(record durableRecord, status string) {
	if s == nil || s.metrics == nil {
		return
	}
	s.metrics.IncCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
		"name":   s.name,
		"kind":   record.Kind,
		"status": status,
	})
}

func (s *DurableSink) recordBacklogAfterReplay() {
	if s == nil || s.metrics == nil {
		return
	}
	if strings.TrimSpace(s.dir) == "" {
		s.recordBacklog(0, 0)
		return
	}
	files, oldestAge, err := durableBacklogStats(s.dir, time.Now())
	if err != nil {
		return
	}
	s.recordBacklog(files, oldestAge)
}

func (s *DurableSink) recordBacklog(files int, oldestAge time.Duration) {
	if s == nil || s.metrics == nil {
		return
	}
	s.metrics.SetGauge("vehicle_durable_spool_backlog_files", metrics.Labels{"name": s.name}, float64(files))
	ageSeconds := 0.0
	if files > 0 && oldestAge > 0 {
		ageSeconds = oldestAge.Seconds()
	}
	s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", metrics.Labels{"name": s.name}, ageSeconds)
}

func durableBacklogStats(dir string, now time.Time) (count int, oldestAge time.Duration, err error) {
	files, err := durableFiles(dir, 0)
	if err != nil {
		return 0, 0, err
	}
	for _, file := range files {
		info, statErr := os.Stat(file)
		if statErr != nil {
			return 0, 0, statErr
		}
		age := now.Sub(info.ModTime())
		if age < 0 {
			age = 0
		}
		if count == 0 || age > oldestAge {
			oldestAge = age
		}
		count++
	}
	return count, oldestAge, nil
}