feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -4,9 +4,14 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +19,7 @@ import (
|
||||
"github.com/segmentio/kafka-go"
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/eventbus"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/health"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/observability"
|
||||
@@ -27,6 +33,10 @@ func main() {
|
||||
defer stop()
|
||||
|
||||
cfg := loadConfig()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
logger.Error("invalid stat writer config", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
db, err := sql.Open("mysql", cfg.MySQLDSN)
|
||||
if err != nil {
|
||||
logger.Error("mysql open failed", "error", err)
|
||||
@@ -38,18 +48,64 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
metrics.RegisterKafkaConsumerInfo(registry, "vehicle-stat-writer", cfg.KafkaGroup, cfg.KafkaTopics)
|
||||
registry.SetGauge("vehicle_stat_project_interval_seconds", nil, cfg.ProjectInterval.Seconds())
|
||||
registry.SetGauge("vehicle_stat_source_touch_interval_seconds", nil, cfg.SourceTouchInterval.Seconds())
|
||||
registry.SetGauge("vehicle_stat_cache_retention_seconds", nil, cfg.CacheRetention.Seconds())
|
||||
registry.SetGauge("vehicle_stat_cache_cleanup_interval_seconds", nil, cfg.CacheCleanupInterval.Seconds())
|
||||
registry.SetGauge("vehicle_stat_baseline_miss_ttl_seconds", nil, cfg.BaselineMissTTL.Seconds())
|
||||
registry.SetGauge("vehicle_stat_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers))
|
||||
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "vehicle-stat-writer", []health.Check{
|
||||
{Name: "mysql", Check: db.PingContext},
|
||||
}, registry))
|
||||
|
||||
writer := stats.NewWriter(db, cfg.Location)
|
||||
writer.SetProjectionInterval(cfg.ProjectInterval)
|
||||
writer.SetSourceTouchInterval(cfg.SourceTouchInterval)
|
||||
writer.SetCacheRetention(cfg.CacheRetention)
|
||||
writer.SetCacheCleanupInterval(cfg.CacheCleanupInterval)
|
||||
writer.SetBaselineMissTTL(cfg.BaselineMissTTL)
|
||||
writer.SetMaxCacheEntries(cfg.CacheMaxEntries)
|
||||
if cfg.EnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx); err != nil {
|
||||
logger.Error("mysql schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
if cfg.NormalizePlatformSourcesOnStart {
|
||||
statDate := time.Now().In(cfg.Location).Format("2006-01-02")
|
||||
normalizeCtx, cancel := context.WithTimeout(ctx, cfg.NormalizePlatformSourcesTimeout)
|
||||
normalized, err := stats.NormalizePlatformSourceMileageForDate(normalizeCtx, db, statDate, envelope.ProtocolJT808)
|
||||
cancel()
|
||||
if err != nil {
|
||||
logger.Warn("platform source startup normalization failed", "stat_date", statDate, "protocol", envelope.ProtocolJT808, "normalized", normalized, "error", err)
|
||||
} else if normalized > 0 {
|
||||
logger.Info("platform source startup normalization finished", "stat_date", statDate, "protocol", envelope.ProtocolJT808, "normalized", normalized)
|
||||
}
|
||||
}
|
||||
var appender statAppender = retryStatAppender{
|
||||
delegate: writer,
|
||||
attempts: cfg.RetryAttempts,
|
||||
delay: cfg.RetryDelay,
|
||||
registry: registry,
|
||||
}
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","), "workers", cfg.Workers, "project_interval_seconds", cfg.ProjectInterval.Seconds(), "source_touch_interval_seconds", cfg.SourceTouchInterval.Seconds(), "cache_retention_seconds", cfg.CacheRetention.Seconds(), "cache_cleanup_interval_seconds", cfg.CacheCleanupInterval.Seconds(), "baseline_miss_ttl_seconds", cfg.BaselineMissTTL.Seconds(), "cache_max_entries", cfg.CacheMaxEntries, "batch_size", cfg.BatchSize, "batch_wait_ms", cfg.BatchWait, "retry_attempts", cfg.RetryAttempts, "retry_delay_ms", cfg.RetryDelay.Milliseconds())
|
||||
var workers sync.WaitGroup
|
||||
for workerID := 1; workerID <= cfg.Workers; workerID++ {
|
||||
workers.Add(1)
|
||||
go func(id int) {
|
||||
defer workers.Done()
|
||||
runStatConsumer(ctx, logger, registry, appender, cfg, id)
|
||||
}(workerID)
|
||||
}
|
||||
workers.Wait()
|
||||
}
|
||||
|
||||
func runStatConsumer(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, cfg config, workerID int) {
|
||||
reader := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: cfg.KafkaBrokers,
|
||||
GroupID: cfg.KafkaGroup,
|
||||
@@ -59,7 +115,10 @@ func main() {
|
||||
})
|
||||
defer reader.Close()
|
||||
|
||||
logger.Info("stat writer started", "group", cfg.KafkaGroup, "topics", strings.Join(cfg.KafkaTopics, ","))
|
||||
workerLabels := metrics.Labels{"worker": strconv.Itoa(workerID)}
|
||||
registry.SetGauge("vehicle_stat_worker_active", workerLabels, 1)
|
||||
defer registry.SetGauge("vehicle_stat_worker_active", workerLabels, 0)
|
||||
|
||||
for {
|
||||
message, err := reader.FetchMessage(ctx)
|
||||
if err != nil {
|
||||
@@ -69,7 +128,8 @@ func main() {
|
||||
logger.Error("kafka fetch failed", "error", err)
|
||||
continue
|
||||
}
|
||||
processStatMessage(ctx, logger, registry, writer, reader, message)
|
||||
batch := collectStatBatch(ctx, reader, message, cfg.BatchSize, time.Duration(cfg.BatchWait)*time.Millisecond)
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, reader, batch, cfg.RetryDelay, workerLabels)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,50 +139,488 @@ type statAppender interface {
|
||||
Append(context.Context, envelope.FrameEnvelope) error
|
||||
}
|
||||
|
||||
type statResultAppender interface {
|
||||
AppendWithResult(context.Context, envelope.FrameEnvelope) (stats.AppendResult, error)
|
||||
}
|
||||
|
||||
type statCacheReporter interface {
|
||||
CacheStats() stats.CacheStats
|
||||
}
|
||||
|
||||
type kafkaMessageCommitter interface {
|
||||
CommitMessages(context.Context, ...kafka.Message) error
|
||||
}
|
||||
|
||||
type kafkaMessageFetcher interface {
|
||||
FetchMessage(context.Context) (kafka.Message, error)
|
||||
}
|
||||
|
||||
type statBatchItem struct {
|
||||
message kafka.Message
|
||||
processed bool
|
||||
}
|
||||
|
||||
type statBatchOutcome struct {
|
||||
commitMessages []kafka.Message
|
||||
retryMessages []kafka.Message
|
||||
}
|
||||
|
||||
var statWriteDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
|
||||
var statWriteE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
|
||||
var statWriteE2ERecent = metrics.NewRecentLatencyByKey(512)
|
||||
|
||||
func collectStatBatch(ctx context.Context, fetcher kafkaMessageFetcher, first kafka.Message, maxSize int, maxWait time.Duration) []kafka.Message {
|
||||
if maxSize <= 1 {
|
||||
return []kafka.Message{first}
|
||||
}
|
||||
if maxWait <= 0 {
|
||||
maxWait = 100 * time.Millisecond
|
||||
}
|
||||
batch := []kafka.Message{first}
|
||||
deadline := time.Now().Add(maxWait)
|
||||
for len(batch) < maxSize {
|
||||
remaining := time.Until(deadline)
|
||||
if remaining <= 0 {
|
||||
break
|
||||
}
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, remaining)
|
||||
message, err := fetcher.FetchMessage(fetchCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
batch = append(batch, message)
|
||||
}
|
||||
return batch
|
||||
}
|
||||
|
||||
func processStatMessage(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, message kafka.Message) {
|
||||
processStatBatch(ctx, logger, registry, appender, committer, []kafka.Message{message})
|
||||
}
|
||||
|
||||
func processStatBatch(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message) statBatchOutcome {
|
||||
return processStatBatchForWorker(ctx, logger, registry, appender, committer, messages, nil)
|
||||
}
|
||||
|
||||
func processStatBatchForWorker(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, workerLabels metrics.Labels) statBatchOutcome {
|
||||
if len(messages) == 0 {
|
||||
return statBatchOutcome{}
|
||||
}
|
||||
messageCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
|
||||
defer cancel()
|
||||
|
||||
addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "received")
|
||||
addStatLagMetric(registry, message)
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "invalid_json")
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
_ = committer.CommitMessages(messageCtx, message)
|
||||
setStatBatchPendingForWorker(registry, workerLabels, len(messages))
|
||||
defer setStatBatchPendingForWorker(registry, workerLabels, 0)
|
||||
defer recordStatCacheMetrics(registry, appender)
|
||||
items := make([]statBatchItem, len(messages))
|
||||
for index, message := range messages {
|
||||
items[index].message = message
|
||||
}
|
||||
for itemIndex, message := range messages {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "received")
|
||||
addStatLagMetric(registry, message)
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(message.Value, &env); err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, "invalid_json")
|
||||
logger.Warn("skip invalid envelope json", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
items[itemIndex].processed = true
|
||||
continue
|
||||
}
|
||||
if status, err := validateStatFieldsEnvelope(message.Topic, env); err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_messages_total", message, status)
|
||||
logger.Warn("skip mismatched stat fields envelope", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "protocol", env.Protocol, "event_id", env.StableEventID(), "event_kind", env.EventKind, "error", err)
|
||||
items[itemIndex].processed = true
|
||||
continue
|
||||
}
|
||||
started := time.Now()
|
||||
result, err := appendStatEnvelope(messageCtx, appender, env)
|
||||
recordStatWriteDuration(registry, message, statusFromError(err), time.Since(started))
|
||||
recordStatSampleMetrics(registry, message, env.Protocol, result)
|
||||
recordStatSourceMetrics(registry, message, env.Protocol, result)
|
||||
recordStatProjectionMetrics(registry, message, env.Protocol, result)
|
||||
if err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_writes_total", message, "error")
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
committed, commitErr := commitStatProcessedPrefixAfterFailure(messageCtx, logger, registry, committer, items)
|
||||
outcome := statBatchOutcome{retryMessages: eventbus.MessagesAfterCommittedPrefixes(messages, committed)}
|
||||
if commitErr != nil {
|
||||
outcome.commitMessages = committed
|
||||
}
|
||||
return outcome
|
||||
}
|
||||
addStatMetric(registry, "vehicle_stat_writes_total", message, "ok")
|
||||
recordStatWriteE2EDuration(registry, message, env)
|
||||
items[itemIndex].processed = true
|
||||
}
|
||||
if err := committer.CommitMessages(messageCtx, messages...); err != nil {
|
||||
for _, message := range messages {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error")
|
||||
}
|
||||
first := messages[0]
|
||||
logger.Error("kafka commit failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err)
|
||||
return statBatchOutcome{commitMessages: messages}
|
||||
}
|
||||
for _, message := range messages {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok")
|
||||
}
|
||||
return statBatchOutcome{}
|
||||
}
|
||||
|
||||
func processStatBatchReliably(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) {
|
||||
processStatBatchReliablyForWorker(ctx, logger, registry, appender, committer, messages, retryDelay, nil)
|
||||
}
|
||||
|
||||
func processStatBatchReliablyForWorker(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, appender statAppender, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration, workerLabels metrics.Labels) {
|
||||
defer registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, 0)
|
||||
pending := messages
|
||||
for len(pending) > 0 {
|
||||
outcome := processStatBatchForWorker(ctx, logger, registry, appender, committer, pending, workerLabels)
|
||||
if len(outcome.commitMessages) > 0 {
|
||||
registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "commit_error"})
|
||||
if !retryStatCommit(ctx, logger, registry, committer, outcome.commitMessages, retryDelay) {
|
||||
return
|
||||
}
|
||||
}
|
||||
pending = outcome.retryMessages
|
||||
registry.SetGauge("vehicle_stat_retry_pending_messages", workerLabels, float64(len(pending)))
|
||||
if len(pending) == 0 {
|
||||
return
|
||||
}
|
||||
registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "write_error"})
|
||||
if !waitForStatRetry(ctx, retryDelay) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func retryStatCommit(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, committer kafkaMessageCommitter, messages []kafka.Message, retryDelay time.Duration) bool {
|
||||
for len(messages) > 0 {
|
||||
operationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), kafkaMessageOperationTimeout)
|
||||
err := committer.CommitMessages(operationCtx, messages...)
|
||||
cancel()
|
||||
if err == nil {
|
||||
for _, message := range messages {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok")
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, message := range messages {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error")
|
||||
}
|
||||
first := messages[0]
|
||||
logger.Error("kafka commit retry failed", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(messages), "error", err)
|
||||
registry.IncCounter("vehicle_stat_batch_retries_total", metrics.Labels{"reason": "commit_error"})
|
||||
if !waitForStatRetry(ctx, retryDelay) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func waitForStatRetry(ctx context.Context, retryDelay time.Duration) bool {
|
||||
if retryDelay <= 0 {
|
||||
retryDelay = 100 * time.Millisecond
|
||||
}
|
||||
timer := time.NewTimer(retryDelay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func validateStatFieldsEnvelope(topic string, env envelope.FrameEnvelope) (string, error) {
|
||||
return topics.ValidateFieldsEnvelope(topic, env)
|
||||
}
|
||||
|
||||
func commitStatProcessedPrefixAfterFailure(ctx context.Context, logger interface {
|
||||
Error(string, ...any)
|
||||
Warn(string, ...any)
|
||||
}, registry *metrics.Registry, committer kafkaMessageCommitter, items []statBatchItem) ([]kafka.Message, error) {
|
||||
committable := processedPrefixMessagesByPartition(items)
|
||||
if len(committable) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := committer.CommitMessages(ctx, committable...); err != nil {
|
||||
for _, message := range committable {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error")
|
||||
}
|
||||
first := committable[0]
|
||||
logger.Error("kafka processed-prefix commit failed after mysql append failure", "topic", first.Topic, "partition", first.Partition, "offset", first.Offset, "messages", len(committable), "error", err)
|
||||
return committable, err
|
||||
}
|
||||
for _, message := range committable {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok")
|
||||
}
|
||||
return committable, nil
|
||||
}
|
||||
|
||||
func processedPrefixMessagesByPartition(items []statBatchItem) []kafka.Message {
|
||||
type partitionKey struct {
|
||||
topic string
|
||||
partition int
|
||||
}
|
||||
groups := map[partitionKey][]statBatchItem{}
|
||||
for _, item := range items {
|
||||
key := partitionKey{topic: item.message.Topic, partition: item.message.Partition}
|
||||
groups[key] = append(groups[key], item)
|
||||
}
|
||||
var keys []partitionKey
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].topic != keys[j].topic {
|
||||
return keys[i].topic < keys[j].topic
|
||||
}
|
||||
return keys[i].partition < keys[j].partition
|
||||
})
|
||||
var out []kafka.Message
|
||||
for _, key := range keys {
|
||||
group := groups[key]
|
||||
sort.Slice(group, func(i, j int) bool {
|
||||
return group[i].message.Offset < group[j].message.Offset
|
||||
})
|
||||
var previousOffset int64
|
||||
for index, item := range group {
|
||||
if index > 0 && item.message.Offset != previousOffset+1 {
|
||||
break
|
||||
}
|
||||
if !item.processed {
|
||||
break
|
||||
}
|
||||
out = append(out, item.message)
|
||||
previousOffset = item.message.Offset
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func appendStatEnvelope(ctx context.Context, appender statAppender, env envelope.FrameEnvelope) (stats.AppendResult, error) {
|
||||
if resultAppender, ok := appender.(statResultAppender); ok {
|
||||
return resultAppender.AppendWithResult(ctx, env)
|
||||
}
|
||||
return stats.AppendResult{}, appender.Append(ctx, env)
|
||||
}
|
||||
|
||||
type retryStatAppender struct {
|
||||
delegate statAppender
|
||||
attempts int
|
||||
delay time.Duration
|
||||
registry *metrics.Registry
|
||||
}
|
||||
|
||||
func (a retryStatAppender) Append(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||
_, err := a.AppendWithResult(ctx, env)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a retryStatAppender) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (stats.AppendResult, error) {
|
||||
if a.delegate == nil {
|
||||
return stats.AppendResult{}, nil
|
||||
}
|
||||
attempts := a.attempts
|
||||
if attempts <= 0 {
|
||||
attempts = 1
|
||||
}
|
||||
var result stats.AppendResult
|
||||
var err error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
result, err = appendStatEnvelope(ctx, a.delegate, env)
|
||||
if err == nil || !isTransientMySQLStatError(err) {
|
||||
return result, err
|
||||
}
|
||||
if attempt == attempts {
|
||||
a.recordRetry("single", "exhausted")
|
||||
return result, err
|
||||
}
|
||||
a.recordRetry("single", "retry")
|
||||
if a.delay <= 0 {
|
||||
continue
|
||||
}
|
||||
timer := time.NewTimer(a.delay)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return result, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (a retryStatAppender) recordRetry(operation string, status string) {
|
||||
if a.registry == nil {
|
||||
return
|
||||
}
|
||||
started := time.Now()
|
||||
err := appender.Append(messageCtx, env)
|
||||
recordStatWriteDuration(registry, message, statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_writes_total", message, "error")
|
||||
logger.Error("mysql append failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "event_id", env.StableEventID(), "error", err)
|
||||
return
|
||||
labels := metrics.Labels{"operation": operation, "status": status}
|
||||
a.registry.IncCounter("vehicle_stat_write_retries_total", labels)
|
||||
metrics.RecordLastActivity(a.registry, "vehicle_stat_last_write_retry_unix_seconds", labels)
|
||||
}
|
||||
|
||||
func (a retryStatAppender) CacheStats() stats.CacheStats {
|
||||
if reporter, ok := a.delegate.(statCacheReporter); ok {
|
||||
return reporter.CacheStats()
|
||||
}
|
||||
addStatMetric(registry, "vehicle_stat_writes_total", message, "ok")
|
||||
if err := committer.CommitMessages(messageCtx, message); err != nil {
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "error")
|
||||
logger.Error("kafka commit failed", "topic", message.Topic, "partition", message.Partition, "offset", message.Offset, "error", err)
|
||||
return
|
||||
return stats.CacheStats{}
|
||||
}
|
||||
|
||||
func isTransientMySQLStatError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
addStatMetric(registry, "vehicle_stat_kafka_commits_total", message, "ok")
|
||||
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
||||
return strings.Contains(text, "deadlock") ||
|
||||
strings.Contains(text, "error 1213") ||
|
||||
strings.Contains(text, "40001") ||
|
||||
strings.Contains(text, "lock wait timeout") ||
|
||||
strings.Contains(text, "error 1205") ||
|
||||
strings.Contains(text, "timeout") ||
|
||||
strings.Contains(text, "temporary") ||
|
||||
strings.Contains(text, "temporarily") ||
|
||||
strings.Contains(text, "connection refused") ||
|
||||
strings.Contains(text, "connection reset") ||
|
||||
strings.Contains(text, "connection closed") ||
|
||||
strings.Contains(text, "broken pipe") ||
|
||||
strings.Contains(text, "bad connection") ||
|
||||
strings.Contains(text, "invalid connection") ||
|
||||
strings.Contains(text, "i/o timeout") ||
|
||||
text == "eof" ||
|
||||
strings.Contains(text, "unexpected eof") ||
|
||||
strings.Contains(text, "server is down") ||
|
||||
strings.Contains(text, "network is unreachable") ||
|
||||
strings.Contains(text, "no route to host")
|
||||
}
|
||||
|
||||
func addStatMetric(registry *metrics.Registry, name string, message kafka.Message, status string) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.IncCounter(name, metrics.Labels{"topic": message.Topic, "status": status})
|
||||
labels := metrics.Labels{"topic": message.Topic, "status": status}
|
||||
registry.IncCounter(name, labels)
|
||||
switch name {
|
||||
case "vehicle_stat_kafka_messages_total":
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_last_message_unix_seconds", labels)
|
||||
case "vehicle_stat_writes_total":
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_last_write_unix_seconds", labels)
|
||||
case "vehicle_stat_kafka_commits_total":
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_last_commit_unix_seconds", labels)
|
||||
}
|
||||
}
|
||||
|
||||
func recordStatSampleMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) {
|
||||
addStatSampleMetric(registry, message, protocol, "found", result.SamplesFound)
|
||||
addStatSampleMetric(registry, message, protocol, "written", result.SamplesWritten)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_fields", result.SamplesSkippedMissingFields)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_vin", result.SamplesSkippedMissingVIN)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_mileage", result.SamplesSkippedMissingMileage)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_non_mileage_frame", result.SamplesSkippedNonMileageFrame)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_non_positive_mileage", result.SamplesSkippedNonPositiveMileage)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_time", result.SamplesSkippedMissingTime)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_same_mileage", result.SamplesSkippedSameMileage)
|
||||
addStatSampleMetric(registry, message, protocol, "skipped_missing_source", result.SamplesSkippedMissingSource)
|
||||
addStatSampleMetric(registry, message, protocol, "event_time_future_adjusted", result.SamplesAdjustedFutureEventTime)
|
||||
}
|
||||
|
||||
func addStatSampleMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) {
|
||||
if registry == nil || count == 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_stat_samples_total", metrics.Labels{
|
||||
"topic": message.Topic,
|
||||
"protocol": statProtocolLabel(protocol),
|
||||
"status": status,
|
||||
}, float64(count))
|
||||
}
|
||||
|
||||
func recordStatSourceMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) {
|
||||
addStatSourceMetric(registry, message, protocol, "attempted", result.SourceTouchesAttempted)
|
||||
addStatSourceMetric(registry, message, protocol, "written", result.SourceTouchesWritten)
|
||||
addStatSourceMetric(registry, message, protocol, "skipped_throttled", result.SourceTouchesSkippedThrottled)
|
||||
addStatSourceMetric(registry, message, protocol, "skipped_missing_endpoint", result.SourceTouchesSkippedMissing)
|
||||
addStatSourceMetric(registry, message, protocol, "skipped_unmanaged", result.SourceTouchesSkippedUnmanaged)
|
||||
}
|
||||
|
||||
func addStatSourceMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) {
|
||||
if registry == nil || count == 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_stat_sources_total", metrics.Labels{
|
||||
"topic": message.Topic,
|
||||
"protocol": statProtocolLabel(protocol),
|
||||
"status": status,
|
||||
}, float64(count))
|
||||
}
|
||||
|
||||
func recordStatProjectionMetrics(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, result stats.AppendResult) {
|
||||
addStatProjectionMetric(registry, message, protocol, "attempted", result.ProjectionsAttempted)
|
||||
addStatProjectionMetric(registry, message, protocol, "written", result.ProjectionsWritten)
|
||||
addStatProjectionMetric(registry, message, protocol, "skipped_throttled", result.ProjectionsSkippedThrottled)
|
||||
}
|
||||
|
||||
func addStatProjectionMetric(registry *metrics.Registry, message kafka.Message, protocol envelope.Protocol, status string, count int) {
|
||||
if registry == nil || count == 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_stat_projections_total", metrics.Labels{
|
||||
"topic": message.Topic,
|
||||
"protocol": statProtocolLabel(protocol),
|
||||
"status": status,
|
||||
}, float64(count))
|
||||
}
|
||||
|
||||
func statProtocolLabel(protocol envelope.Protocol) string {
|
||||
protocolLabel := strings.TrimSpace(string(protocol))
|
||||
if protocolLabel == "" {
|
||||
return "UNKNOWN"
|
||||
}
|
||||
return protocolLabel
|
||||
}
|
||||
|
||||
func recordStatCacheMetrics(registry *metrics.Registry, appender statAppender) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
reporter, ok := appender.(statCacheReporter)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
stats := reporter.CacheStats()
|
||||
setStatCacheGauge(registry, "last_total_mileage", stats.LastTotalMileageEntries, stats.MaxEntries, stats.LastCleanupTotalMileage, stats.TotalMileageEvictions)
|
||||
setStatCacheGauge(registry, "source_seen", stats.LastSourceSeenEntries, stats.MaxEntries, stats.LastCleanupSourceSeen, stats.SourceSeenEvictions)
|
||||
setStatCacheGauge(registry, "projection", stats.LastProjectionEntries, stats.MaxEntries, stats.LastCleanupProjection, stats.ProjectionEvictions)
|
||||
setStatCacheGauge(registry, "baseline", stats.BaselineEntries, stats.MaxEntries, stats.LastCleanupBaseline, stats.BaselineEvictions)
|
||||
if !stats.LastCleanupAt.IsZero() {
|
||||
registry.SetGauge("vehicle_stat_cache_last_cleanup_unix_seconds", nil, float64(stats.LastCleanupAt.Unix()))
|
||||
}
|
||||
}
|
||||
|
||||
func setStatCacheGauge(registry *metrics.Registry, cache string, entries int, maxEntries int, lastCleanupDeleted int, evictions int) {
|
||||
labels := metrics.Labels{"cache": cache}
|
||||
registry.SetGauge("vehicle_stat_cache_entries", labels, float64(entries))
|
||||
registry.SetGauge("vehicle_stat_cache_max_entries", labels, float64(maxEntries))
|
||||
registry.SetGauge("vehicle_stat_cache_last_cleanup_deleted", labels, float64(lastCleanupDeleted))
|
||||
registry.SetGauge("vehicle_stat_cache_evictions_total", labels, float64(evictions))
|
||||
}
|
||||
|
||||
func addStatLagMetric(registry *metrics.Registry, message kafka.Message) {
|
||||
@@ -142,6 +640,33 @@ func recordStatWriteDuration(registry *metrics.Registry, message kafka.Message,
|
||||
}, statWriteDurationBucketsMS, float64(elapsed.Milliseconds()))
|
||||
}
|
||||
|
||||
func recordStatWriteE2EDuration(registry *metrics.Registry, message kafka.Message, env envelope.FrameEnvelope) {
|
||||
if registry == nil || env.ReceivedAtMS <= 0 {
|
||||
return
|
||||
}
|
||||
elapsed := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds()
|
||||
if elapsed < 0 {
|
||||
elapsed = 0
|
||||
}
|
||||
labels := metrics.Labels{"topic": message.Topic}
|
||||
registry.ObserveHistogram("vehicle_stat_write_e2e_duration_ms_histogram", labels, statWriteE2EDurationBucketsMS, float64(elapsed))
|
||||
p99, samples := statWriteE2ERecent.Observe(message.Topic, float64(elapsed))
|
||||
registry.SetGauge("vehicle_stat_write_e2e_recent_p99_ms", labels, p99)
|
||||
registry.SetGauge("vehicle_stat_write_e2e_recent_samples", labels, float64(samples))
|
||||
metrics.RecordLastActivity(registry, "vehicle_stat_last_write_e2e_unix_seconds", labels)
|
||||
}
|
||||
|
||||
func setStatBatchPending(registry *metrics.Registry, messages int) {
|
||||
setStatBatchPendingForWorker(registry, nil, messages)
|
||||
}
|
||||
|
||||
func setStatBatchPendingForWorker(registry *metrics.Registry, workerLabels metrics.Labels, messages int) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.SetGauge("vehicle_stat_batch_pending_messages", workerLabels, float64(messages))
|
||||
}
|
||||
|
||||
func statusFromError(err error) string {
|
||||
if err != nil {
|
||||
return "error"
|
||||
@@ -150,12 +675,40 @@ func statusFromError(err error) string {
|
||||
}
|
||||
|
||||
type config struct {
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
MySQLDSN string
|
||||
EnsureSchema bool
|
||||
Location *time.Location
|
||||
KafkaBrokers []string
|
||||
KafkaTopics []string
|
||||
KafkaGroup string
|
||||
MySQLDSN string
|
||||
EnsureSchema bool
|
||||
Location *time.Location
|
||||
ProjectInterval time.Duration
|
||||
SourceTouchInterval time.Duration
|
||||
CacheRetention time.Duration
|
||||
CacheCleanupInterval time.Duration
|
||||
BaselineMissTTL time.Duration
|
||||
CacheMaxEntries int
|
||||
Workers int
|
||||
BatchSize int
|
||||
BatchWait int
|
||||
RetryAttempts int
|
||||
RetryDelay time.Duration
|
||||
NormalizePlatformSourcesOnStart bool
|
||||
NormalizePlatformSourcesTimeout time.Duration
|
||||
}
|
||||
|
||||
func (c config) Validate() error {
|
||||
if len(c.KafkaTopics) == 0 {
|
||||
return fmt.Errorf("KAFKA_TOPICS must include fields topics")
|
||||
}
|
||||
for _, topic := range c.KafkaTopics {
|
||||
if !strings.HasPrefix(topic, "vehicle.fields.") {
|
||||
return fmt.Errorf("stat-writer consumes fields topics only, got %q", topic)
|
||||
}
|
||||
}
|
||||
if c.Workers <= 0 {
|
||||
return fmt.Errorf("STATS_WORKERS must be greater than zero")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
@@ -164,12 +717,25 @@ func loadConfig() config {
|
||||
loc = time.FixedZone("Asia/Shanghai", 8*3600)
|
||||
}
|
||||
return config{
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT}, ","))),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||
Location: loc,
|
||||
KafkaBrokers: splitCSV(env("KAFKA_BROKERS", "127.0.0.1:9092")),
|
||||
KafkaTopics: splitCSV(env("KAFKA_TOPICS", strings.Join([]string{topics.FieldsGB32960, topics.FieldsJT808, topics.FieldsYutongMQTT}, ","))),
|
||||
KafkaGroup: env("KAFKA_GROUP", "go-stat-writer"),
|
||||
MySQLDSN: env("MYSQL_DSN", ""),
|
||||
EnsureSchema: env("MYSQL_ENSURE_SCHEMA", "true") != "false",
|
||||
Location: loc,
|
||||
ProjectInterval: time.Duration(envInt("STATS_PROJECT_INTERVAL_SECONDS", 15)) * time.Second,
|
||||
SourceTouchInterval: time.Duration(envInt("STATS_SOURCE_TOUCH_INTERVAL_SECONDS", 60)) * time.Second,
|
||||
CacheRetention: time.Duration(envInt("STATS_CACHE_RETENTION_HOURS", 72)) * time.Hour,
|
||||
CacheCleanupInterval: time.Duration(envInt("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", 600)) * time.Second,
|
||||
BaselineMissTTL: time.Duration(envInt("STATS_BASELINE_MISS_TTL_SECONDS", 60)) * time.Second,
|
||||
CacheMaxEntries: envInt("STATS_CACHE_MAX_ENTRIES", 1000000),
|
||||
Workers: envInt("STATS_WORKERS", 3),
|
||||
BatchSize: envInt("STATS_BATCH_SIZE", 200),
|
||||
BatchWait: envInt("STATS_BATCH_WAIT_MS", 20),
|
||||
RetryAttempts: envInt("STATS_RETRY_ATTEMPTS", 3),
|
||||
RetryDelay: time.Duration(envInt("STATS_RETRY_DELAY_MS", 20)) * time.Millisecond,
|
||||
NormalizePlatformSourcesOnStart: env("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "true") != "false",
|
||||
NormalizePlatformSourcesTimeout: time.Duration(envInt("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", 30)) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,6 +747,18 @@ func env(key string, fallback string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
func envInt(key string, fallback int) int {
|
||||
value := strings.TrimSpace(os.Getenv(key))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
|
||||
Reference in New Issue
Block a user