shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: spool gateway publishes to disk

Browse Source
This commit is contained in:
lingniu
2026-07-01 23:00:06 +08:00
parent 4a32e2181a
commit 243de5c2b2
4 changed files with 360 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
go/vehicle-gateway/cmd/gateway/main.go
View File

@@ -27,7 +27,7 @@ func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
sink, err := buildSink(logger)
sink, err := buildSink(ctx, logger)
if err != nil {
logger.Error("build sink failed", "error", err)
os.Exit(1)
@@ -132,7 +132,7 @@ func buildIdentityResolver(ctx context.Context, logger *slog.Logger) (identity.R
return identity.NewMySQLResolver(db, table), func() { _ = db.Close() }, nil
}
func buildSink(logger *slog.Logger) (eventbus.Sink, error) {
func buildSink(ctx context.Context, logger *slog.Logger) (eventbus.Sink, error) {
brokers := splitCSV(os.Getenv("KAFKA_BROKERS"))
if len(brokers) == 0 {
logger.Warn("KAFKA_BROKERS is empty; using log sink")
@@ -150,11 +150,22 @@ func buildSink(logger *slog.Logger) (eventbus.Sink, error) {
if err != nil {
return nil, err
}
return eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
var out eventbus.Sink = eventbus.NewRetryingSink(sink, eventbus.RetryConfig{
Attempts: envInt("KAFKA_PUBLISH_ATTEMPTS", 3),
Backoff: time.Duration(envInt("KAFKA_PUBLISH_BACKOFF_MS", 100)) * time.Millisecond,
AttemptTimeout: time.Duration(envInt("KAFKA_PUBLISH_TIMEOUT_MS", 3000)) * time.Millisecond,
}), nil
})
spoolDir := strings.TrimSpace(os.Getenv("KAFKA_SPOOL_DIR"))
if spoolDir != "" {
durable := eventbus.NewDurableSink(out, eventbus.DurableConfig{Directory: spoolDir})
interval := time.Duration(envInt("KAFKA_SPOOL_REPLAY_INTERVAL_MS", 1000)) * time.Millisecond
go durable.ReplayLoop(ctx, interval, func(err error) {
logger.Warn("kafka spool replay failed", "error", err)
})
logger.Info("kafka durable spool enabled", "dir", spoolDir, "replay_interval_ms", interval.Milliseconds())
out = durable
}
return out, nil
}
func env(key string, fallback string) string {