shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
go/vehicle-gateway/internal/eventbus/durable_sink.go
View File

@@ -13,16 +13,21 @@ import (
"time"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
)
type DurableConfig struct {
Directory string
ReplayBatchSize int
Metrics *metrics.Registry
Name string
}
type DurableSink struct {
delegate Sink
dir string
metrics *metrics.Registry
name string
mu sync.Mutex
seq uint64
@@ -50,12 +55,16 @@ func NewDurableSink(delegate Sink, cfg DurableConfig) *DurableSink {
if delegate == nil {
panic("durable delegate sink must not be nil")
}
return &DurableSink{
s := &DurableSink{
delegate: delegate,
dir: strings.TrimSpace(cfg.Directory),
metrics: cfg.Metrics,
name: durableMetricName(cfg.Name),
rawPending: map[string]struct{}{},
replayBatchSize: cfg.ReplayBatchSize,
}
s.recordBacklogAfterReplay()
return s
}
func (s *DurableSink) PublishRaw(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -96,16 +105,29 @@ func (s *DurableSink) ReplayOnce(ctx context.Context) error {
func (s *DurableSink) replay(ctx context.Context, limit int) error {
files, err := durableFiles(s.dir, limit)
if err != nil {
s.recordReplay("list_error", 0)
return err
}
s.recordBacklogAfterReplay()
records := make([]durableRecordFile, 0, len(files))
for _, file := range files {
record, err := readDurableRecord(file)
if err != nil {
return err
s.recordReplay("read_error", 1)
if quarantineErr := quarantineDurableFile(file); quarantineErr != nil {
s.recordReplay("quarantine_error", 1)
return fmt.Errorf("quarantine unreadable durable record %s: read error: %w; quarantine error: %v", file, err, quarantineErr)
}
s.recordReplay("quarantined", 1)
continue
}
records = append(records, durableRecordFile{path: file, record: record})
}
if len(records) == 0 {
s.recordReplay("empty", 0)
s.recordBacklogAfterReplay()
return nil
}
sortDurableRecords(records)
if publisher, ok := s.delegate.(recordPublishingSink); ok {
durableRecords := make([]durableRecord, 0, len(records))
@@ -113,29 +135,41 @@ func (s *DurableSink) replay(ctx context.Context, limit int) error {
durableRecords = append(durableRecords, item.record)
}
if err := publisher.PublishRecords(ctx, durableRecords); err != nil {
s.recordReplay("publish_error", len(records))
s.recordReplayRecords(records, "publish_error")
return err
}
for _, item := range records {
if err := os.Remove(item.path); err != nil {
s.recordReplay("delete_error", len(records))
return err
}
if item.record.Kind == "raw" {
s.clearRawPending(item.record.Envelope)
}
}
s.recordReplay("ok", len(records))
s.recordReplayRecords(records, "ok")
s.recordBacklogAfterReplay()
return nil
}
for _, item := range records {
if err := s.publishRecord(ctx, item.record); err != nil {
s.recordReplay("publish_error", len(records))
s.recordReplayRecord(item.record, "publish_error")
return err
}
if err := os.Remove(item.path); err != nil {
s.recordReplay("delete_error", len(records))
return err
}
if item.record.Kind == "raw" {
s.clearRawPending(item.record.Envelope)
}
s.recordReplayRecord(item.record, "ok")
}
s.recordReplay("ok", len(records))
s.recordBacklogAfterReplay()
return nil
}
@@ -210,7 +244,7 @@ func errUnknownRecordKind(kind string) error {
func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
if s.dir == "" {
return fmt.Errorf("durable spool directory is empty")
return s.spoolError(kind, fmt.Errorf("durable spool directory is empty"))
}
if env.EventID == "" {
env.EventID = env.StableEventID()
@@ -219,19 +253,24 @@ func (s *DurableSink) spool(kind string, env envelope.FrameEnvelope) error {
env.ParseStatus = envelope.ParseOK
}
if err := os.MkdirAll(s.dir, 0o750); err != nil {
return err
return s.spoolError(kind, err)
}
payload, err := json.Marshal(durableRecord{Kind: kind, Envelope: env})
if err != nil {
return err
return s.spoolError(kind, err)
}
name := s.nextFileName(env, kind)
path := filepath.Join(s.dir, name)
tmp := path + ".tmp"
if err := os.WriteFile(tmp, payload, 0o640); err != nil {
return err
return s.spoolError(kind, err)
}
return os.Rename(tmp, path)
if err := os.Rename(tmp, path); err != nil {
return s.spoolError(kind, err)
}
s.recordSpool(kind, "ok")
s.recordBacklogAfterReplay()
return nil
}
func (s *DurableSink) nextFileName(env envelope.FrameEnvelope, kind string) string {
@@ -273,6 +312,16 @@ func readDurableRecord(path string) (durableRecord, error) {
return record, json.Unmarshal(payload, &record)
}
func quarantineDurableFile(path string) error {
target := path + ".bad"
if _, err := os.Stat(target); err == nil {
target = fmt.Sprintf("%s.%d.bad", path, time.Now().UnixNano())
} else if !os.IsNotExist(err) {
return err
}
return os.Rename(path, target)
}
func durableFiles(dir string, limit int) ([]string, error) {
if limit > 0 {
handle, err := os.Open(dir)
@@ -330,3 +379,108 @@ func durableFilesFromReader(dir string, limit int, reader durableNameReader) ([]
func errorsIsEOF(err error) bool {
return err == io.EOF
}
func durableMetricName(name string) string {
name = strings.TrimSpace(name)
if name == "" {
return "default"
}
return name
}
func (s *DurableSink) spoolError(kind string, err error) error {
s.recordSpool(kind, "error")
return err
}
func (s *DurableSink) recordSpool(kind string, status string) {
if s == nil || s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_spool_records_total", metrics.Labels{
"name": s.name,
"kind": kind,
"status": status,
})
}
func (s *DurableSink) recordReplay(status string, records int) {
if s == nil || s.metrics == nil {
return
}
labels := metrics.Labels{"name": s.name, "status": status}
s.metrics.IncCounter("vehicle_durable_spool_replay_total", labels)
if records > 0 {
s.metrics.AddCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
"name": s.name,
"kind": "all",
"status": status,
}, float64(records))
}
}
func (s *DurableSink) recordReplayRecords(records []durableRecordFile, status string) {
for _, item := range records {
s.recordReplayRecord(item.record, status)
}
}
func (s *DurableSink) recordReplayRecord(record durableRecord, status string) {
if s == nil || s.metrics == nil {
return
}
s.metrics.IncCounter("vehicle_durable_spool_replay_records_total", metrics.Labels{
"name": s.name,
"kind": record.Kind,
"status": status,
})
}
func (s *DurableSink) recordBacklogAfterReplay() {
if s == nil || s.metrics == nil {
return
}
if strings.TrimSpace(s.dir) == "" {
s.recordBacklog(0, 0)
return
}
files, oldestAge, err := durableBacklogStats(s.dir, time.Now())
if err != nil {
return
}
s.recordBacklog(files, oldestAge)
}
func (s *DurableSink) recordBacklog(files int, oldestAge time.Duration) {
if s == nil || s.metrics == nil {
return
}
s.metrics.SetGauge("vehicle_durable_spool_backlog_files", metrics.Labels{"name": s.name}, float64(files))
ageSeconds := 0.0
if files > 0 && oldestAge > 0 {
ageSeconds = oldestAge.Seconds()
}
s.metrics.SetGauge("vehicle_durable_spool_oldest_age_seconds", metrics.Labels{"name": s.name}, ageSeconds)
}
func durableBacklogStats(dir string, now time.Time) (count int, oldestAge time.Duration, err error) {
files, err := durableFiles(dir, 0)
if err != nil {
return 0, 0, err
}
for _, file := range files {
info, statErr := os.Stat(file)
if statErr != nil {
return 0, 0, statErr
}
age := now.Sub(info.ModTime())
if age < 0 {
age = 0
}
if count == 0 || age > oldestAge {
oldestAge = age
}
count++
}
return count, oldestAge, nil
}