package eventbus import ( "context" "encoding/json" "fmt" "io" "os" "path/filepath" "sort" "strings" "sync" "time" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" ) type DurableConfig struct { Directory string ReplayBatchSize int Metrics *metrics.Registry Name string } type DurableSink struct { delegate Sink dir string metrics *metrics.Registry name string mu sync.Mutex seq uint64 rawPending map[string]struct{} replayBatchSize int } type durableRecord struct { Kind string `json:"kind"` Envelope envelope.FrameEnvelope `json:"envelope"` } type durableRecordFile struct { path string record durableRecord } type recordPublishingSink interface { PublishRecords(context.Context, []durableRecord) error } const durableReplayOperationTimeout = 30 * time.Second func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink { if delegate == nil { panic("durable delegate sink must not be nil") } s := &DurableSink{ delegate: delegate, dir: strings.TrimSpace(cfg.Directory), metrics: cfg.Metrics, name: durableMetricName(cfg.Name), rawPending: map[string]struct{}{}, replayBatchSize: cfg.ReplayBatchSize, } s.recordBacklogAfterReplay() return s } func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error { if err := s.delegate.PublishRaw(ctx, env); err == nil { return nil } if err := s.spool("raw", env); err != nil { return err } s.markRawPending(env) return nil } func (s *DurableSink) PublishUnified(ctx context.Context, env envelope.FrameEnvelope) error { if s.isRawPending(env) { return s.spool("unified", env) } if err := s.delegate.PublishUnified(ctx, env); err == nil { return nil } return s.spool("unified", env) } func (s *DurableSink) PublishFields(ctx context.Context, env envelope.FrameEnvelope) error { if s.isRawPending(env) { return s.spool("fields", env) } if err := s.delegate.PublishFields(ctx, env); err == nil { return nil } return s.spool("fields", env) } func (s *DurableSink) ReplayOnce(ctx context.Context) error { return s.replay(ctx, 0) } func (s *DurableSink) replay(ctx context.Context, limit int) error { files, err := durableFiles(s.dir, limit) if err != nil { s.recordReplay("list_error", 0) return err } s.recordBacklogAfterReplay() records := make([]durableRecordFile, 0, len(files)) for _, file := range files { record, err := readDurableRecord(file) if err != nil { s.recordReplay("read_error", 1) if quarantineErr := quarantineDurableFile(file); quarantineErr != nil { s.recordReplay("quarantine_error", 1) return fmt.Errorf("quarantine unreadable durable record %s: read error: %w; quarantine error: %v", file, err, quarantineErr) } s.recordReplay("quarantined", 1) continue } records = append(records, durableRecordFile{path: file, record: record}) } if len(records) == 0 { s.recordReplay("empty", 0) s.recordBacklogAfterReplay() return nil } sortDurableRecords(records) if publisher, ok := s.delegate.(recordPublishingSink); ok { durableRecords := make([]durableRecord, 0, len(records)) for _, item := range records { durableRecords = append(durableRecords, item.record) } if err := publisher.PublishRecords(ctx, durableRecords); err != nil { s.recordReplay("publish_error", len(records)) s.recordReplayRecords(records, "publish_error") return err } for _, item := range records { if err := os.Remove(item.path); err != nil { s.recordReplay("delete_error", len(records)) return err } if item.record.Kind == "raw" { s.clearRawPending(item.record.Envelope) } } s.recordReplay("ok", len(records)) s.recordReplayRecords(records, "ok") s.recordBacklogAfterReplay() return nil } for _, item := range records { if err := s.publishRecord(ctx, item.record); err != nil { s.recordReplay("publish_error", len(records)) s.recordReplayRecord(item.record, "publish_error") return err } if err := os.Remove(item.path); err != nil { s.recordReplay("delete_error", len(records)) return err } if item.record.Kind == "raw" { s.clearRawPending(item.record.Envelope) } s.recordReplayRecord(item.record, "ok") } s.recordReplay("ok", len(records)) s.recordBacklogAfterReplay() return nil } func sortDurableRecords(records []durableRecordFile) { sort.SliceStable(records, func(i, j int) bool { leftEvent := records[i].record.Envelope.StableEventID() rightEvent := records[j].record.Envelope.StableEventID() if leftEvent == rightEvent { return durableKindOrder(records[i].record.Kind) < durableKindOrder(records[j].record.Kind) } return records[i].path < records[j].path }) } func durableKindOrder(kind string) int { if kind == "raw" { return 0 } if kind == "unified" { return 1 } if kind == "fields" { return 2 } return 3 } func (s *DurableSink) ReplayLoop(ctx context.Context, interval time.Duration, onError func(error)) { if interval <= 0 { interval = time.Second } ticker := time.NewTicker(interval) defer ticker.Stop() for { select { case <-ctx.Done(): return case <-ticker.C: if err := s.replayOnceFromLoop(ctx); err != nil && onError != nil { onError(err) } } } } func (s *DurableSink) replayOnceFromLoop(ctx context.Context) error { replayCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), durableReplayOperationTimeout) defer cancel() return s.replay(replayCtx, s.replayBatchSize) } func (s *DurableSink) Close() error { return s.delegate.Close() } func (s *DurableSink) publishRecord(ctx context.Context, record durableRecord) error { switch record.Kind { case "raw": return s.delegate.PublishRaw(ctx, record.Envelope) case "unified": return s.delegate.PublishUnified(ctx, record.Envelope) case "fields": return s.delegate.PublishFields(ctx, record.Envelope) default: return errUnknownRecordKind(record.Kind) } } func errUnknownRecordKind(kind string) error { return fmt.Errorf("unknown durable record kind %q", kind) } func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error { if s.dir == "" { return s.spoolError(kind, fmt.Errorf("durable spool directory is empty")) } if env.EventID == "" { env.EventID = env.StableEventID() } if env.ParseStatus == "" { env.ParseStatus = envelope.ParseOK } if err := os.MkdirAll(s.dir, 0o750); err != nil { return s.spoolError(kind, err) } payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env}) if err != nil { return s.spoolError(kind, err) } name := s.nextFileName(env, kind) path := filepath.Join(s.dir, name) tmp := path + ".tmp" if err := os.WriteFile(tmp, payload, 0o640); err != nil { return s.spoolError(kind, err) } if err := os.Rename(tmp, path); err != nil { return s.spoolError(kind, err) } s.recordSpool(kind, "ok") s.recordBacklogAfterReplay() return nil } func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string { s.mu.Lock() defer s.mu.Unlock() s.seq++ eventID := env.StableEventID() if len(eventID) > 12 { eventID = eventID[:12] } return fmt.Sprintf("%020d-%06d-%s-%s.json", time.Now().UnixNano(), s.seq, kind, eventID) } func (s *DurableSink) markRawPending(env envelope.FrameEnvelope) { s.mu.Lock() defer s.mu.Unlock() s.rawPending[env.StableEventID()] = struct{}{} } func (s *DurableSink) clearRawPending(env envelope.FrameEnvelope) { s.mu.Lock() defer s.mu.Unlock() delete(s.rawPending, env.StableEventID()) } func (s *DurableSink) isRawPending(env envelope.FrameEnvelope) bool { s.mu.Lock() defer s.mu.Unlock() _, ok := s.rawPending[env.StableEventID()] return ok } func readDurableRecord(path string) (durableRecord, error) { var record durableRecord payload, err := os.ReadFile(path) if err != nil { return record, err } return record, json.Unmarshal(payload, &record) } func quarantineDurableFile(path string) error { target := path + ".bad" if _, err := os.Stat(target); err == nil { target = fmt.Sprintf("%s.%d.bad", path, time.Now().UnixNano()) } else if !os.IsNotExist(err) { return err } return os.Rename(path, target) } func durableFiles(dir string, limit int) ([]string, error) { if limit > 0 { handle, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { return nil, nil } return nil, err } defer handle.Close() return durableFilesFromReader(dir, limit, handle) } files, err := filepath.Glob(filepath.Join(dir, "*.json")) if err != nil { return nil, err } sort.Strings(files) return files, nil } type durableNameReader interface { Readdirnames(int) ([]string, error) } func durableFilesFromReader(dir string, limit int, reader durableNameReader) ([]string, error) { if limit <= 0 { return nil, nil } files := make([]string, 0, limit) for len(files) < limit { names, err := reader.Readdirnames(limit - len(files)) for _, name := range names { if !strings.HasSuffix(name, ".json") { continue } files = append(files, filepath.Join(dir, name)) if len(files) >= limit { break } } if err != nil { if errorsIsEOF(err) { break } return nil, err } if len(names) == 0 { break } } sort.Strings(files) return files, nil } func errorsIsEOF(err error) bool { return err == io.EOF } func durableMetricName(name string) string { name = strings.TrimSpace(name) if name == "" { return "default" } return name } func (s *DurableSink) spoolError(kind string, err error) error { s.recordSpool(kind, "error") return err } func (s *DurableSink) recordSpool(kind string, status string) { if s == nil || s.metrics == nil { return } s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{ "name": s.name, "kind": kind, "status": status, }) } func (s *DurableSink) recordReplay(status string, records int) { if s == nil || s.metrics == nil { return } labels := metrics.Labels{"name": s.name, "status": status} s.metrics.IncCounter("vehicle_durable_spool_replay_total", labels) if records > 0 { s.metrics.AddCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{ "name": s.name, "kind": "all", "status": status, }, float64(records)) } } func (s *DurableSink) recordReplayRecords(records []durableRecordFile, status string) { for _, item := range records { s.recordReplayRecord(item.record, status) } } func (s *DurableSink) recordReplayRecord(record durableRecord, status string) { if s == nil || s.metrics == nil { return } s.metrics.IncCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{ "name": s.name, "kind": record.Kind, "status": status, }) } func (s *DurableSink) recordBacklogAfterReplay() { if s == nil || s.metrics == nil { return } if strings.TrimSpace(s.dir) == "" { s.recordBacklog(0, 0) return } files, oldestAge, err := durableBacklogStats(s.dir, time.Now()) if err != nil { return } s.recordBacklog(files, oldestAge) } func (s *DurableSink) recordBacklog(files int, oldestAge time.Duration) { if s == nil || s.metrics == nil { return } s.metrics.SetGauge("vehicle_durable_spool_backlog_files", metrics.Labels{"name": s.name}, float64(files)) ageSeconds := 0.0 if files > 0 && oldestAge > 0 { ageSeconds = oldestAge.Seconds() } s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", metrics.Labels{"name": s.name}, ageSeconds) } func durableBacklogStats(dir string, now time.Time) (count int, oldestAge time.Duration, err error) { files, err := durableFiles(dir, 0) if err != nil { return 0, 0, err } for _, file := range files { info, statErr := os.Stat(file) if statErr != nil { return 0, 0, statErr } age := now.Sub(info.ModTime()) if age < 0 { age = 0 } if count == 0 || age > oldestAge { oldestAge = age } count++ } return count, oldestAge, nil }