shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

579
go/vehicle-gateway/cmd/nats-fast-writer/main.go
View File

@@ -60,24 +60,32 @@ func main() {
os.Exit(1)
}
tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
if err != nil {
logger.Error("tdengine open failed", "error", err)
os.Exit(1)
}
defer tdDB.Close()
tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns)
tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns)
if err := tdDB.PingContext(ctx); err != nil {
logger.Error("tdengine ping failed", "error", err)
os.Exit(1)
}
historyWriter := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase)
if cfg.TDengineEnsureSchema {
if err := historyWriter.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
logger.Error("tdengine schema bootstrap failed", "error", err)
var historyWriter fastAppender
var tdCheck health.Check
if cfg.TDengineEnabled {
tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
if err != nil {
logger.Error("tdengine open failed", "error", err)
os.Exit(1)
}
defer tdDB.Close()
tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns)
tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns)
if err := tdDB.PingContext(ctx); err != nil {
logger.Error("tdengine ping failed", "error", err)
os.Exit(1)
}
writer := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase)
if cfg.TDengineEnsureSchema {
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
logger.Error("tdengine schema bootstrap failed", "error", err)
os.Exit(1)
}
}
historyWriter = writer
tdCheck = health.Check{Name: "tdengine", Check: tdDB.PingContext}
} else {
logger.Info("nats fast writer tdengine stage disabled")
}
redisClient := redis.NewClient(&redis.Options{
@@ -94,23 +102,30 @@ func main() {
realtimeRepo := realtime.NewRepository(redisClient, realtime.Config{OnlineTTL: cfg.OnlineTTL})
registry := metrics.NewRegistry()
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", []health.Check{
recordFastWriterConfigMetrics(registry, cfg)
registry.SetGauge("vehicle_fast_writer_tdengine_enabled", nil, boolGauge(cfg.TDengineEnabled))
healthChecks := []health.Check{
{Name: "nats", Check: func(context.Context) error {
if conn.Status() != nats.CONNECTED {
return fmt.Errorf("nats status is %s", conn.Status().String())
}
return nil
}},
{Name: "tdengine", Check: tdDB.PingContext},
{Name: "redis", Check: func(ctx context.Context) error { return redisClient.Ping(ctx).Err() }},
}, registry))
}
if tdCheck.Name != "" {
healthChecks = append(healthChecks, tdCheck)
}
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", healthChecks, registry))
logger.Info("nats fast writer started",
"stream", cfg.NATSStream,
"durable", cfg.NATSDurable,
"filter", cfg.NATSFilter,
"batch_size", cfg.BatchSize,
"fetch_wait_ms", cfg.FetchWait.Milliseconds(),
"workers", cfg.Workers,
"tdengine_enabled", cfg.TDengineEnabled,
"tdengine_max_open_conns", cfg.TDengineMaxOpenConns,
"tdengine_max_idle_conns", cfg.TDengineMaxIdleConns,
"operation_timeout_ms", cfg.OperationWait.Milliseconds())
@@ -135,6 +150,7 @@ type config struct {
StreamMaxBytes int64
StreamEnsureWait time.Duration
Workers int
TDengineEnabled bool
TDengineDriver string
TDengineDSN string
TDengineDatabase string
@@ -168,13 +184,14 @@ func loadConfig() config {
NATSFilter: env("NATS_FILTER", "vehicle.raw.go.>"),
NATSSubjects: subjects,
BatchSize: envInt("FAST_WRITER_BATCH_SIZE", 100),
FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 100)) * time.Millisecond,
FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 20)) * time.Millisecond,
OperationWait: time.Duration(envInt("FAST_WRITER_OPERATION_TIMEOUT_MS", 1000)) * time.Millisecond,
AckWait: time.Duration(envInt("NATS_ACK_WAIT_SECONDS", 30)) * time.Second,
StreamMaxAge: time.Duration(envInt("NATS_STREAM_MAX_AGE_HOURS", 24)) * time.Hour,
StreamMaxBytes: envInt64("NATS_STREAM_MAX_BYTES", 20*1024*1024*1024),
StreamEnsureWait: time.Duration(envInt("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", 60)) * time.Second,
Workers: workers,
TDengineEnabled: envBool("FAST_WRITER_TDENGINE_ENABLED", false),
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
TDengineDSN: env("TDENGINE_DSN", ""),
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
@@ -198,10 +215,18 @@ type fastUpdater interface {
FastUpdate(context.Context, envelope.FrameEnvelope) error
}
type fastResultUpdater interface {
FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error)
}
type fastBatchUpdater interface {
FastUpdateBatch(context.Context, []envelope.FrameEnvelope) error
}
type fastBatchResultUpdater interface {
FastUpdateBatchWithResult(context.Context, []envelope.FrameEnvelope) (realtime.FastUpdateResult, error)
}
type fastMessage struct {
subject string
data []byte
@@ -227,9 +252,17 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R
}
msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait))
if err != nil {
if isFastWorkerShutdownFetchError(ctx, err) {
return
}
if errors.Is(err, nats.ErrTimeout) {
continue
}
if isTransientFastFetchError(err) {
logger.Warn("nats fetch interrupted", "error", err)
time.Sleep(time.Second)
continue
}
logger.Error("nats fetch failed", "error", err)
time.Sleep(time.Second)
continue
@@ -253,12 +286,33 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R
logger.Error("fast write batch failed", "messages", len(fastMessages), "error", err)
continue
}
for _, msg := range fastMessages {
addFastMetric(registry, msg.subject, "ok")
}
}
}
func isFastWorkerShutdownFetchError(ctx context.Context, err error) bool {
if err == nil {
return false
}
if ctx.Err() != nil {
return true
}
return errors.Is(err, nats.ErrConnectionClosed)
}
func isTransientFastFetchError(err error) bool {
if err == nil {
return false
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
return strings.Contains(text, "disconnected during fetch") ||
strings.Contains(text, "connection closed") ||
strings.Contains(text, "connection reset") ||
strings.Contains(text, "broken pipe") ||
strings.Contains(text, "temporary") ||
strings.Contains(text, "temporarily") ||
strings.Contains(text, "timeout")
}
type natsPullSubscription interface {
Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
}
@@ -268,18 +322,44 @@ type natsConsumerInfoReader interface {
}
var fastWriterStageDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
var fastWriterRedisE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
var fastWriterRedisE2ERecent = metrics.NewRecentLatencyByKey(512)
var fastBatchPending = metrics.PendingPairGauge{}
func processFastBatch(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error {
if len(messages) == 0 {
return nil
}
for _, msg := range messages {
addFastMetric(registry, msg.subject, "received")
}
envelopes := make([]envelope.FrameEnvelope, 0, len(messages))
validMessages := make([]*fastMessage, 0, len(messages))
for _, msg := range messages {
var env envelope.FrameEnvelope
if err := json.Unmarshal(msg.data, &env); err != nil {
addFastMetric(registry, msg.subject, "invalid_json")
if msg.ack != nil {
_ = msg.ack()
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack invalid json: %w", err)
}
}
continue
}
if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil {
addFastMetric(registry, msg.subject, status)
if msg.ack != nil {
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack mismatched raw envelope: %w", err)
}
}
continue
}
@@ -289,85 +369,358 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender
if len(envelopes) == 0 {
return nil
}
setFastBatchPending(registry, len(messages), len(envelopes))
defer setFastBatchPending(registry, 0, 0)
addFastBatchPending(registry, len(messages), len(envelopes))
defer addFastBatchPending(registry, -len(messages), -len(envelopes))
subject := fastBatchSubject(validMessages)
started := time.Now()
err := appender.AppendAllBatch(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("tdengine batch append: %w", err)
}
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
started = time.Now()
err = batchUpdater.FastUpdateBatch(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
if appender != nil {
started := time.Now()
err := appender.AppendAllBatch(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("redis fast batch update: %w", err)
if shouldFallbackFastBatchError(err) {
if fallbackErr := processFastMessagesIndividually(ctx, registry, appender, updater, validMessages); fallbackErr != nil {
addFastBatchFallbackMetric(registry, "tdengine", "error")
return fmt.Errorf("tdengine batch fallback: %w", fallbackErr)
}
addFastBatchFallbackMetric(registry, "tdengine", "ok")
return nil
}
addFastBatchFallbackMetric(registry, "tdengine", "skipped_transient")
if catchupErr := updateFastRedisBatchWithoutAck(ctx, registry, updater, validMessages, envelopes); catchupErr != nil {
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error")
return fmt.Errorf("tdengine batch append: %w; redis catchup: %v", err, catchupErr)
}
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok")
return fmt.Errorf("tdengine batch append: %w", err)
}
}
if batchUpdater, ok := updater.(fastBatchResultUpdater); ok {
for _, group := range fastSubjectGroups(validMessages, envelopes) {
started := time.Now()
result, err := batchUpdater.FastUpdateBatchWithResult(ctx, group.envelopes)
recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
if shouldFallbackFastBatchError(err) {
if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil {
addFastBatchFallbackMetric(registry, "redis", "error")
return fmt.Errorf("redis fast batch fallback: %w", fallbackErr)
}
addFastBatchFallbackMetric(registry, "redis", "ok")
return nil
}
addFastBatchFallbackMetric(registry, "redis", "skipped_transient")
return fmt.Errorf("redis fast batch update: %w", err)
}
recordFastWriterRedisEnvelopeMetrics(registry, group.subject, result)
recordFastWriterRedisFieldMetrics(registry, group.subject, result)
recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject)
}
for _, msg := range validMessages {
if msg.ack != nil {
started = time.Now()
err = msg.ack()
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack: %w", err)
}
}
addFastMetric(registry, msg.subject, "ok")
}
return nil
}
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
for _, group := range fastSubjectGroups(validMessages, envelopes) {
started := time.Now()
err := batchUpdater.FastUpdateBatch(ctx, group.envelopes)
recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
if shouldFallbackFastBatchError(err) {
if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil {
addFastBatchFallbackMetric(registry, "redis", "error")
return fmt.Errorf("redis fast batch fallback: %w", fallbackErr)
}
addFastBatchFallbackMetric(registry, "redis", "ok")
return nil
}
addFastBatchFallbackMetric(registry, "redis", "skipped_transient")
return fmt.Errorf("redis fast batch update: %w", err)
}
recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject)
}
for _, msg := range validMessages {
if msg.ack != nil {
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack: %w", err)
}
}
addFastMetric(registry, msg.subject, "ok")
}
return nil
}
for i, env := range envelopes {
msg := validMessages[i]
started = time.Now()
err = updater.FastUpdate(ctx, env)
started := time.Now()
var result realtime.FastUpdateResult
var err error
if resultUpdater, ok := updater.(fastResultUpdater); ok {
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
} else {
err = updater.FastUpdate(ctx, env)
}
recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("redis fast update: %w", err)
}
recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result)
recordFastWriterRedisFieldMetrics(registry, msg.subject, result)
recordFastWriterRedisE2EDuration(registry, msg.subject, env)
if msg.ack != nil {
started = time.Now()
err = msg.ack()
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack: %w", err)
}
}
addFastMetric(registry, msg.subject, "ok")
}
return nil
}
func processFastMessagesIndividually(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error {
for _, msg := range messages {
if err := processFastMessageWithReceived(ctx, registry, appender, updater, msg, false); err != nil {
return err
}
}
return nil
}
func processFastMessage(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage) error {
return processFastMessageWithReceived(ctx, registry, appender, updater, msg, true)
}
func processFastMessageWithReceived(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage, recordReceived bool) error {
if recordReceived {
addFastMetric(registry, msg.subject, "received")
}
var env envelope.FrameEnvelope
if err := json.Unmarshal(msg.data, &env); err != nil {
addFastMetric(registry, msg.subject, "invalid_json")
if msg.ack != nil {
_ = msg.ack()
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack invalid json: %w", err)
}
}
return nil
}
started := time.Now()
err := appender.AppendAll(ctx, env)
recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("tdengine append: %w", err)
if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil {
addFastMetric(registry, msg.subject, status)
if msg.ack != nil {
started := time.Now()
err := msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack mismatched raw envelope: %w", err)
}
}
return nil
}
if appender != nil {
started := time.Now()
err := appender.AppendAll(ctx, env)
recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started))
if err != nil {
if isTransientFastBatchError(err) {
if catchupErr := updateFastRedisSingleWithoutAck(ctx, registry, updater, msg.subject, env); catchupErr != nil {
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error")
return fmt.Errorf("tdengine append: %w; redis catchup: %v", err, catchupErr)
}
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok")
}
return fmt.Errorf("tdengine append: %w", err)
}
}
started := time.Now()
var result realtime.FastUpdateResult
var err error
if resultUpdater, ok := updater.(fastResultUpdater); ok {
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
} else {
err = updater.FastUpdate(ctx, env)
}
started = time.Now()
err = updater.FastUpdate(ctx, env)
recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
return fmt.Errorf("redis fast update: %w", err)
}
recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result)
recordFastWriterRedisFieldMetrics(registry, msg.subject, result)
recordFastWriterRedisE2EDuration(registry, msg.subject, env)
if msg.ack != nil {
started = time.Now()
err = msg.ack()
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
if err != nil {
addFastMetric(registry, msg.subject, "ack_error")
return fmt.Errorf("nats ack: %w", err)
}
}
addFastMetric(registry, msg.subject, "ok")
return nil
}
func updateFastRedisBatchWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error {
if len(envelopes) == 0 {
return nil
}
subject := fastBatchSubject(messages)
if batchUpdater, ok := updater.(fastBatchResultUpdater); ok {
started := time.Now()
result, err := batchUpdater.FastUpdateBatchWithResult(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
if err == nil {
recordFastWriterRedisEnvelopeMetrics(registry, subject, result)
recordFastWriterRedisFieldMetrics(registry, subject, result)
recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject)
return nil
}
if shouldFallbackFastBatchError(err) {
addFastBatchFallbackMetric(registry, "redis_catchup", "attempted")
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
}
return err
}
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
started := time.Now()
err := batchUpdater.FastUpdateBatch(ctx, envelopes)
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
if err == nil {
recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject)
return nil
}
if shouldFallbackFastBatchError(err) {
addFastBatchFallbackMetric(registry, "redis_catchup", "attempted")
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
}
return err
}
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
}
func updateFastRedisSinglesWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error {
for index, env := range envelopes {
subject := fastMessageSubject(messages, index, fastBatchSubject(messages))
if err := updateFastRedisSingleWithoutAck(ctx, registry, updater, subject, env); err != nil {
return err
}
}
return nil
}
func updateFastRedisSingleWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, subject string, env envelope.FrameEnvelope) error {
started := time.Now()
var result realtime.FastUpdateResult
var err error
if resultUpdater, ok := updater.(fastResultUpdater); ok {
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
} else {
err = updater.FastUpdate(ctx, env)
}
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
if err != nil {
return err
}
recordFastWriterRedisEnvelopeMetrics(registry, subject, result)
recordFastWriterRedisFieldMetrics(registry, subject, result)
recordFastWriterRedisE2EDuration(registry, subject, env)
return nil
}
func shouldFallbackFastBatchError(err error) bool {
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return false
}
return !isTransientFastBatchError(err)
}
func isTransientFastBatchError(err error) bool {
if err == nil {
return false
}
text := strings.ToLower(strings.TrimSpace(err.Error()))
return strings.Contains(text, "timeout") ||
strings.Contains(text, "temporary") ||
strings.Contains(text, "temporarily") ||
strings.Contains(text, "connection refused") ||
strings.Contains(text, "connection reset") ||
strings.Contains(text, "connection closed") ||
strings.Contains(text, "broken pipe") ||
strings.Contains(text, "bad connection") ||
strings.Contains(text, "i/o timeout") ||
text == "eof" ||
strings.Contains(text, "unexpected eof") ||
strings.Contains(text, "server is down") ||
strings.Contains(text, "network is unreachable") ||
strings.Contains(text, "no route to host")
}
func recordFastWriterConfigMetrics(registry *metrics.Registry, cfg config) {
if registry == nil {
return
}
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers))
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize))
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "fetch_wait_ms"}, float64(cfg.FetchWait.Milliseconds()))
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationWait.Milliseconds()))
}
func boolGauge(value bool) float64 {
if value {
return 1
}
return 0
}
type fastSubjectGroup struct {
subject string
messages []*fastMessage
envelopes []envelope.FrameEnvelope
}
func fastSubjectGroups(messages []*fastMessage, envelopes []envelope.FrameEnvelope) []fastSubjectGroup {
if len(envelopes) == 0 {
return nil
}
fallback := fastBatchSubject(messages)
groups := make([]fastSubjectGroup, 0, len(envelopes))
indexBySubject := make(map[string]int, len(envelopes))
for index, env := range envelopes {
subject := fastMessageSubject(messages, index, fallback)
groupIndex, exists := indexBySubject[subject]
if !exists {
groupIndex = len(groups)
indexBySubject[subject] = groupIndex
groups = append(groups, fastSubjectGroup{subject: subject})
}
groups[groupIndex].envelopes = append(groups[groupIndex].envelopes, env)
if index >= 0 && index < len(messages) {
groups[groupIndex].messages = append(groups[groupIndex].messages, messages[index])
}
}
return groups
}
func fastBatchSubject(messages []*fastMessage) string {
if len(messages) == 0 {
return "unknown"
@@ -384,6 +737,16 @@ func fastBatchSubject(messages []*fastMessage) string {
return subject
}
func fastMessageSubject(messages []*fastMessage, index int, fallback string) string {
if index >= 0 && index < len(messages) && strings.TrimSpace(messages[index].subject) != "" {
return messages[index].subject
}
if strings.TrimSpace(fallback) != "" {
return fallback
}
return "unknown"
}
func ensureStream(js nats.JetStreamContext, cfg config) error {
ctx, cancel := context.WithTimeout(context.Background(), cfg.StreamEnsureWait)
defer cancel()
@@ -416,15 +779,36 @@ func addFastMetric(registry *metrics.Registry, subject string, status string) {
if registry == nil {
return
}
registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status})
labels := metrics.Labels{"subject": subject, "status": status}
registry.IncCounter("vehicle_fast_writer_messages_total", labels)
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_message_unix_seconds", labels)
}
func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int) {
func addFastBatchFallbackMetric(registry *metrics.Registry, stage string, status string) {
if registry == nil {
return
}
registry.SetGauge("vehicle_fast_writer_batch_pending_messages", nil, float64(messages))
registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes))
registry.IncCounter("vehicle_fast_writer_batch_fallback_total", metrics.Labels{
"stage": stage,
"status": status,
})
}
func addFastDecoupledUpdateMetric(registry *metrics.Registry, reason string, status string) {
if registry == nil {
return
}
registry.IncCounter("vehicle_fast_writer_decoupled_updates_total", metrics.Labels{
"reason": reason,
"status": status,
})
}
func addFastBatchPending(registry *metrics.Registry, messages int, envelopes int) {
if registry == nil {
return
}
fastBatchPending.Add(registry, "vehicle_fast_writer_batch_pending_messages", "vehicle_fast_writer_batch_pending_envelopes", messages, envelopes)
}
func recordFastNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) {
@@ -441,11 +825,78 @@ func recordFastWriterStageDuration(registry *metrics.Registry, subject string, s
if registry == nil {
return
}
registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", metrics.Labels{
labels := metrics.Labels{
"subject": subject,
"stage": stage,
"status": status,
}, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds()))
}
registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", labels, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds()))
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_stage_unix_seconds", labels)
}
func recordFastWriterRedisE2EDurationMessages(registry *metrics.Registry, messages []*fastMessage, envelopes []envelope.FrameEnvelope, fallback string) {
for index, env := range envelopes {
recordFastWriterRedisE2EDuration(registry, fastMessageSubject(messages, index, fallback), env)
}
}
func recordFastWriterRedisE2EDuration(registry *metrics.Registry, subject string, env envelope.FrameEnvelope) {
if registry == nil || env.ReceivedAtMS <= 0 {
return
}
elapsedMS := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds()
if elapsedMS < 0 {
elapsedMS = 0
}
labels := metrics.Labels{
"subject": subject,
}
registry.ObserveHistogram("vehicle_fast_writer_redis_e2e_duration_ms_histogram", labels, fastWriterRedisE2EDurationBucketsMS, float64(elapsedMS))
p99, samples := fastWriterRedisE2ERecent.Observe(subject, float64(elapsedMS))
registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_p99_ms", labels, p99)
registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_samples", labels, float64(samples))
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_redis_e2e_unix_seconds", labels)
}
func recordFastWriterRedisFieldMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) {
if registry == nil || result.FieldsSeen == 0 {
return
}
addFastWriterRedisFieldMetric(registry, subject, "seen", result.FieldsSeen)
addFastWriterRedisFieldMetric(registry, subject, "written", result.FieldsWritten)
addFastWriterRedisFieldMetric(registry, subject, "skipped_stale", result.FieldsSkippedStale)
}
func recordFastWriterRedisEnvelopeMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) {
if registry == nil || result.EnvelopesSeen == 0 {
return
}
addFastWriterRedisEnvelopeMetric(registry, subject, "seen", result.EnvelopesSeen)
addFastWriterRedisEnvelopeMetric(registry, subject, "updated", result.EnvelopesUpdated)
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_non_realtime", result.EnvelopesSkippedNonRealtime)
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vin", result.EnvelopesSkippedMissingVIN)
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vehicle_key", result.EnvelopesSkippedMissingVehicleKey)
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_fields", result.EnvelopesSkippedMissingFields)
}
func addFastWriterRedisEnvelopeMetric(registry *metrics.Registry, subject string, status string, value int) {
if value <= 0 {
return
}
registry.AddCounter("vehicle_fast_writer_redis_envelopes_total", metrics.Labels{
"subject": subject,
"status": status,
}, float64(value))
}
func addFastWriterRedisFieldMetric(registry *metrics.Registry, subject string, status string, value int) {
if value <= 0 {
return
}
registry.AddCounter("vehicle_fast_writer_redis_fields_total", metrics.Labels{
"subject": subject,
"status": status,
}, float64(value))
}
func statusFromError(err error) string {
@@ -487,6 +938,20 @@ func envInt64(key string, fallback int64) int64 {
return parsed
}
func envBool(key string, fallback bool) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
switch value {
case "":
return fallback
case "1", "true", "yes", "y", "on":
return true
case "0", "false", "no", "n", "off":
return false
default:
return fallback
}
}
func splitCSV(value string) []string {
var out []string
for _, item := range strings.Split(value, ",") {