feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -60,24 +60,32 @@ func main() {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||
if err != nil {
|
||||
logger.Error("tdengine open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns)
|
||||
tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns)
|
||||
if err := tdDB.PingContext(ctx); err != nil {
|
||||
logger.Error("tdengine ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
historyWriter := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase)
|
||||
if cfg.TDengineEnsureSchema {
|
||||
if err := historyWriter.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||
var historyWriter fastAppender
|
||||
var tdCheck health.Check
|
||||
if cfg.TDengineEnabled {
|
||||
tdDB, err := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN)
|
||||
if err != nil {
|
||||
logger.Error("tdengine open failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer tdDB.Close()
|
||||
tdDB.SetMaxOpenConns(cfg.TDengineMaxOpenConns)
|
||||
tdDB.SetMaxIdleConns(cfg.TDengineMaxIdleConns)
|
||||
if err := tdDB.PingContext(ctx); err != nil {
|
||||
logger.Error("tdengine ping failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
writer := history.NewWriterWithDatabase(tdDB, cfg.TDengineDatabase)
|
||||
if cfg.TDengineEnsureSchema {
|
||||
if err := writer.EnsureSchema(ctx, cfg.TDengineDatabase); err != nil {
|
||||
logger.Error("tdengine schema bootstrap failed", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
historyWriter = writer
|
||||
tdCheck = health.Check{Name: "tdengine", Check: tdDB.PingContext}
|
||||
} else {
|
||||
logger.Info("nats fast writer tdengine stage disabled")
|
||||
}
|
||||
|
||||
redisClient := redis.NewClient(&redis.Options{
|
||||
@@ -94,23 +102,30 @@ func main() {
|
||||
realtimeRepo := realtime.NewRepository(redisClient, realtime.Config{OnlineTTL: cfg.OnlineTTL})
|
||||
|
||||
registry := metrics.NewRegistry()
|
||||
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", []health.Check{
|
||||
recordFastWriterConfigMetrics(registry, cfg)
|
||||
registry.SetGauge("vehicle_fast_writer_tdengine_enabled", nil, boolGauge(cfg.TDengineEnabled))
|
||||
healthChecks := []health.Check{
|
||||
{Name: "nats", Check: func(context.Context) error {
|
||||
if conn.Status() != nats.CONNECTED {
|
||||
return fmt.Errorf("nats status is %s", conn.Status().String())
|
||||
}
|
||||
return nil
|
||||
}},
|
||||
{Name: "tdengine", Check: tdDB.PingContext},
|
||||
{Name: "redis", Check: func(ctx context.Context) error { return redisClient.Ping(ctx).Err() }},
|
||||
}, registry))
|
||||
}
|
||||
if tdCheck.Name != "" {
|
||||
healthChecks = append(healthChecks, tdCheck)
|
||||
}
|
||||
health.Start(ctx, logger, health.NewServer(env("HEALTH_ADDR", ""), "nats-fast-writer", healthChecks, registry))
|
||||
|
||||
logger.Info("nats fast writer started",
|
||||
"stream", cfg.NATSStream,
|
||||
"durable", cfg.NATSDurable,
|
||||
"filter", cfg.NATSFilter,
|
||||
"batch_size", cfg.BatchSize,
|
||||
"fetch_wait_ms", cfg.FetchWait.Milliseconds(),
|
||||
"workers", cfg.Workers,
|
||||
"tdengine_enabled", cfg.TDengineEnabled,
|
||||
"tdengine_max_open_conns", cfg.TDengineMaxOpenConns,
|
||||
"tdengine_max_idle_conns", cfg.TDengineMaxIdleConns,
|
||||
"operation_timeout_ms", cfg.OperationWait.Milliseconds())
|
||||
@@ -135,6 +150,7 @@ type config struct {
|
||||
StreamMaxBytes int64
|
||||
StreamEnsureWait time.Duration
|
||||
Workers int
|
||||
TDengineEnabled bool
|
||||
TDengineDriver string
|
||||
TDengineDSN string
|
||||
TDengineDatabase string
|
||||
@@ -168,13 +184,14 @@ func loadConfig() config {
|
||||
NATSFilter: env("NATS_FILTER", "vehicle.raw.go.>"),
|
||||
NATSSubjects: subjects,
|
||||
BatchSize: envInt("FAST_WRITER_BATCH_SIZE", 100),
|
||||
FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 100)) * time.Millisecond,
|
||||
FetchWait: time.Duration(envInt("FAST_WRITER_FETCH_WAIT_MS", 20)) * time.Millisecond,
|
||||
OperationWait: time.Duration(envInt("FAST_WRITER_OPERATION_TIMEOUT_MS", 1000)) * time.Millisecond,
|
||||
AckWait: time.Duration(envInt("NATS_ACK_WAIT_SECONDS", 30)) * time.Second,
|
||||
StreamMaxAge: time.Duration(envInt("NATS_STREAM_MAX_AGE_HOURS", 24)) * time.Hour,
|
||||
StreamMaxBytes: envInt64("NATS_STREAM_MAX_BYTES", 20*1024*1024*1024),
|
||||
StreamEnsureWait: time.Duration(envInt("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", 60)) * time.Second,
|
||||
Workers: workers,
|
||||
TDengineEnabled: envBool("FAST_WRITER_TDENGINE_ENABLED", false),
|
||||
TDengineDriver: env("TDENGINE_DRIVER", "taosWS"),
|
||||
TDengineDSN: env("TDENGINE_DSN", ""),
|
||||
TDengineDatabase: env("TDENGINE_DATABASE", history.DefaultDatabase),
|
||||
@@ -198,10 +215,18 @@ type fastUpdater interface {
|
||||
FastUpdate(context.Context, envelope.FrameEnvelope) error
|
||||
}
|
||||
|
||||
type fastResultUpdater interface {
|
||||
FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error)
|
||||
}
|
||||
|
||||
type fastBatchUpdater interface {
|
||||
FastUpdateBatch(context.Context, []envelope.FrameEnvelope) error
|
||||
}
|
||||
|
||||
type fastBatchResultUpdater interface {
|
||||
FastUpdateBatchWithResult(context.Context, []envelope.FrameEnvelope) (realtime.FastUpdateResult, error)
|
||||
}
|
||||
|
||||
type fastMessage struct {
|
||||
subject string
|
||||
data []byte
|
||||
@@ -227,9 +252,17 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R
|
||||
}
|
||||
msgs, err := sub.Fetch(cfg.BatchSize, nats.MaxWait(cfg.FetchWait))
|
||||
if err != nil {
|
||||
if isFastWorkerShutdownFetchError(ctx, err) {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, nats.ErrTimeout) {
|
||||
continue
|
||||
}
|
||||
if isTransientFastFetchError(err) {
|
||||
logger.Warn("nats fetch interrupted", "error", err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
logger.Error("nats fetch failed", "error", err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
@@ -253,12 +286,33 @@ func runFastWorker(ctx context.Context, logger *slog.Logger, registry *metrics.R
|
||||
logger.Error("fast write batch failed", "messages", len(fastMessages), "error", err)
|
||||
continue
|
||||
}
|
||||
for _, msg := range fastMessages {
|
||||
addFastMetric(registry, msg.subject, "ok")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isFastWorkerShutdownFetchError(ctx context.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return true
|
||||
}
|
||||
return errors.Is(err, nats.ErrConnectionClosed)
|
||||
}
|
||||
|
||||
func isTransientFastFetchError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
||||
return strings.Contains(text, "disconnected during fetch") ||
|
||||
strings.Contains(text, "connection closed") ||
|
||||
strings.Contains(text, "connection reset") ||
|
||||
strings.Contains(text, "broken pipe") ||
|
||||
strings.Contains(text, "temporary") ||
|
||||
strings.Contains(text, "temporarily") ||
|
||||
strings.Contains(text, "timeout")
|
||||
}
|
||||
|
||||
type natsPullSubscription interface {
|
||||
Fetch(int, ...nats.PullOpt) ([]*nats.Msg, error)
|
||||
}
|
||||
@@ -268,18 +322,44 @@ type natsConsumerInfoReader interface {
|
||||
}
|
||||
|
||||
var fastWriterStageDurationBucketsMS = []float64{1, 5, 10, 25, 50, 100, 250, 500, 1000, 5000}
|
||||
var fastWriterRedisE2EDurationBucketsMS = []float64{10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000}
|
||||
var fastWriterRedisE2ERecent = metrics.NewRecentLatencyByKey(512)
|
||||
var fastBatchPending = metrics.PendingPairGauge{}
|
||||
|
||||
func processFastBatch(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error {
|
||||
if len(messages) == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, msg := range messages {
|
||||
addFastMetric(registry, msg.subject, "received")
|
||||
}
|
||||
envelopes := make([]envelope.FrameEnvelope, 0, len(messages))
|
||||
validMessages := make([]*fastMessage, 0, len(messages))
|
||||
for _, msg := range messages {
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(msg.data, &env); err != nil {
|
||||
addFastMetric(registry, msg.subject, "invalid_json")
|
||||
if msg.ack != nil {
|
||||
_ = msg.ack()
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack invalid json: %w", err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil {
|
||||
addFastMetric(registry, msg.subject, status)
|
||||
if msg.ack != nil {
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack mismatched raw envelope: %w", err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -289,85 +369,358 @@ func processFastBatch(ctx context.Context, registry *metrics.Registry, appender
|
||||
if len(envelopes) == 0 {
|
||||
return nil
|
||||
}
|
||||
setFastBatchPending(registry, len(messages), len(envelopes))
|
||||
defer setFastBatchPending(registry, 0, 0)
|
||||
addFastBatchPending(registry, len(messages), len(envelopes))
|
||||
defer addFastBatchPending(registry, -len(messages), -len(envelopes))
|
||||
subject := fastBatchSubject(validMessages)
|
||||
started := time.Now()
|
||||
err := appender.AppendAllBatch(ctx, envelopes)
|
||||
recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return fmt.Errorf("tdengine batch append: %w", err)
|
||||
}
|
||||
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
|
||||
started = time.Now()
|
||||
err = batchUpdater.FastUpdateBatch(ctx, envelopes)
|
||||
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
|
||||
if appender != nil {
|
||||
started := time.Now()
|
||||
err := appender.AppendAllBatch(ctx, envelopes)
|
||||
recordFastWriterStageDuration(registry, subject, "tdengine", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis fast batch update: %w", err)
|
||||
if shouldFallbackFastBatchError(err) {
|
||||
if fallbackErr := processFastMessagesIndividually(ctx, registry, appender, updater, validMessages); fallbackErr != nil {
|
||||
addFastBatchFallbackMetric(registry, "tdengine", "error")
|
||||
return fmt.Errorf("tdengine batch fallback: %w", fallbackErr)
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "tdengine", "ok")
|
||||
return nil
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "tdengine", "skipped_transient")
|
||||
if catchupErr := updateFastRedisBatchWithoutAck(ctx, registry, updater, validMessages, envelopes); catchupErr != nil {
|
||||
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error")
|
||||
return fmt.Errorf("tdengine batch append: %w; redis catchup: %v", err, catchupErr)
|
||||
}
|
||||
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok")
|
||||
return fmt.Errorf("tdengine batch append: %w", err)
|
||||
}
|
||||
}
|
||||
if batchUpdater, ok := updater.(fastBatchResultUpdater); ok {
|
||||
for _, group := range fastSubjectGroups(validMessages, envelopes) {
|
||||
started := time.Now()
|
||||
result, err := batchUpdater.FastUpdateBatchWithResult(ctx, group.envelopes)
|
||||
recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
if shouldFallbackFastBatchError(err) {
|
||||
if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil {
|
||||
addFastBatchFallbackMetric(registry, "redis", "error")
|
||||
return fmt.Errorf("redis fast batch fallback: %w", fallbackErr)
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "redis", "ok")
|
||||
return nil
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "redis", "skipped_transient")
|
||||
return fmt.Errorf("redis fast batch update: %w", err)
|
||||
}
|
||||
recordFastWriterRedisEnvelopeMetrics(registry, group.subject, result)
|
||||
recordFastWriterRedisFieldMetrics(registry, group.subject, result)
|
||||
recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject)
|
||||
}
|
||||
for _, msg := range validMessages {
|
||||
if msg.ack != nil {
|
||||
started = time.Now()
|
||||
err = msg.ack()
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack: %w", err)
|
||||
}
|
||||
}
|
||||
addFastMetric(registry, msg.subject, "ok")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
|
||||
for _, group := range fastSubjectGroups(validMessages, envelopes) {
|
||||
started := time.Now()
|
||||
err := batchUpdater.FastUpdateBatch(ctx, group.envelopes)
|
||||
recordFastWriterStageDuration(registry, group.subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
if shouldFallbackFastBatchError(err) {
|
||||
if fallbackErr := processFastMessagesIndividually(ctx, registry, nil, updater, validMessages); fallbackErr != nil {
|
||||
addFastBatchFallbackMetric(registry, "redis", "error")
|
||||
return fmt.Errorf("redis fast batch fallback: %w", fallbackErr)
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "redis", "ok")
|
||||
return nil
|
||||
}
|
||||
addFastBatchFallbackMetric(registry, "redis", "skipped_transient")
|
||||
return fmt.Errorf("redis fast batch update: %w", err)
|
||||
}
|
||||
recordFastWriterRedisE2EDurationMessages(registry, group.messages, group.envelopes, group.subject)
|
||||
}
|
||||
for _, msg := range validMessages {
|
||||
if msg.ack != nil {
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack: %w", err)
|
||||
}
|
||||
}
|
||||
addFastMetric(registry, msg.subject, "ok")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for i, env := range envelopes {
|
||||
msg := validMessages[i]
|
||||
started = time.Now()
|
||||
err = updater.FastUpdate(ctx, env)
|
||||
started := time.Now()
|
||||
var result realtime.FastUpdateResult
|
||||
var err error
|
||||
if resultUpdater, ok := updater.(fastResultUpdater); ok {
|
||||
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
|
||||
} else {
|
||||
err = updater.FastUpdate(ctx, env)
|
||||
}
|
||||
recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis fast update: %w", err)
|
||||
}
|
||||
recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result)
|
||||
recordFastWriterRedisFieldMetrics(registry, msg.subject, result)
|
||||
recordFastWriterRedisE2EDuration(registry, msg.subject, env)
|
||||
if msg.ack != nil {
|
||||
started = time.Now()
|
||||
err = msg.ack()
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack: %w", err)
|
||||
}
|
||||
}
|
||||
addFastMetric(registry, msg.subject, "ok")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processFastMessagesIndividually(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, messages []*fastMessage) error {
|
||||
for _, msg := range messages {
|
||||
if err := processFastMessageWithReceived(ctx, registry, appender, updater, msg, false); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func processFastMessage(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage) error {
|
||||
return processFastMessageWithReceived(ctx, registry, appender, updater, msg, true)
|
||||
}
|
||||
|
||||
func processFastMessageWithReceived(ctx context.Context, registry *metrics.Registry, appender fastAppender, updater fastUpdater, msg *fastMessage, recordReceived bool) error {
|
||||
if recordReceived {
|
||||
addFastMetric(registry, msg.subject, "received")
|
||||
}
|
||||
var env envelope.FrameEnvelope
|
||||
if err := json.Unmarshal(msg.data, &env); err != nil {
|
||||
addFastMetric(registry, msg.subject, "invalid_json")
|
||||
if msg.ack != nil {
|
||||
_ = msg.ack()
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack invalid json: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
started := time.Now()
|
||||
err := appender.AppendAll(ctx, env)
|
||||
recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return fmt.Errorf("tdengine append: %w", err)
|
||||
if status, err := topics.ValidateRawEnvelope(msg.subject, env); err != nil {
|
||||
addFastMetric(registry, msg.subject, status)
|
||||
if msg.ack != nil {
|
||||
started := time.Now()
|
||||
err := msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack mismatched raw envelope: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if appender != nil {
|
||||
started := time.Now()
|
||||
err := appender.AppendAll(ctx, env)
|
||||
recordFastWriterStageDuration(registry, msg.subject, "tdengine", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
if isTransientFastBatchError(err) {
|
||||
if catchupErr := updateFastRedisSingleWithoutAck(ctx, registry, updater, msg.subject, env); catchupErr != nil {
|
||||
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "error")
|
||||
return fmt.Errorf("tdengine append: %w; redis catchup: %v", err, catchupErr)
|
||||
}
|
||||
addFastDecoupledUpdateMetric(registry, "tdengine_transient", "ok")
|
||||
}
|
||||
return fmt.Errorf("tdengine append: %w", err)
|
||||
}
|
||||
}
|
||||
started := time.Now()
|
||||
var result realtime.FastUpdateResult
|
||||
var err error
|
||||
if resultUpdater, ok := updater.(fastResultUpdater); ok {
|
||||
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
|
||||
} else {
|
||||
err = updater.FastUpdate(ctx, env)
|
||||
}
|
||||
started = time.Now()
|
||||
err = updater.FastUpdate(ctx, env)
|
||||
recordFastWriterStageDuration(registry, msg.subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return fmt.Errorf("redis fast update: %w", err)
|
||||
}
|
||||
recordFastWriterRedisEnvelopeMetrics(registry, msg.subject, result)
|
||||
recordFastWriterRedisFieldMetrics(registry, msg.subject, result)
|
||||
recordFastWriterRedisE2EDuration(registry, msg.subject, env)
|
||||
if msg.ack != nil {
|
||||
started = time.Now()
|
||||
err = msg.ack()
|
||||
recordFastWriterStageDuration(registry, msg.subject, "ack", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
addFastMetric(registry, msg.subject, "ack_error")
|
||||
return fmt.Errorf("nats ack: %w", err)
|
||||
}
|
||||
}
|
||||
addFastMetric(registry, msg.subject, "ok")
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateFastRedisBatchWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error {
|
||||
if len(envelopes) == 0 {
|
||||
return nil
|
||||
}
|
||||
subject := fastBatchSubject(messages)
|
||||
if batchUpdater, ok := updater.(fastBatchResultUpdater); ok {
|
||||
started := time.Now()
|
||||
result, err := batchUpdater.FastUpdateBatchWithResult(ctx, envelopes)
|
||||
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err == nil {
|
||||
recordFastWriterRedisEnvelopeMetrics(registry, subject, result)
|
||||
recordFastWriterRedisFieldMetrics(registry, subject, result)
|
||||
recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject)
|
||||
return nil
|
||||
}
|
||||
if shouldFallbackFastBatchError(err) {
|
||||
addFastBatchFallbackMetric(registry, "redis_catchup", "attempted")
|
||||
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if batchUpdater, ok := updater.(fastBatchUpdater); ok {
|
||||
started := time.Now()
|
||||
err := batchUpdater.FastUpdateBatch(ctx, envelopes)
|
||||
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err == nil {
|
||||
recordFastWriterRedisE2EDurationMessages(registry, messages, envelopes, subject)
|
||||
return nil
|
||||
}
|
||||
if shouldFallbackFastBatchError(err) {
|
||||
addFastBatchFallbackMetric(registry, "redis_catchup", "attempted")
|
||||
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return updateFastRedisSinglesWithoutAck(ctx, registry, updater, messages, envelopes)
|
||||
}
|
||||
|
||||
func updateFastRedisSinglesWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, messages []*fastMessage, envelopes []envelope.FrameEnvelope) error {
|
||||
for index, env := range envelopes {
|
||||
subject := fastMessageSubject(messages, index, fastBatchSubject(messages))
|
||||
if err := updateFastRedisSingleWithoutAck(ctx, registry, updater, subject, env); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func updateFastRedisSingleWithoutAck(ctx context.Context, registry *metrics.Registry, updater fastUpdater, subject string, env envelope.FrameEnvelope) error {
|
||||
started := time.Now()
|
||||
var result realtime.FastUpdateResult
|
||||
var err error
|
||||
if resultUpdater, ok := updater.(fastResultUpdater); ok {
|
||||
result, err = resultUpdater.FastUpdateWithResult(ctx, env)
|
||||
} else {
|
||||
err = updater.FastUpdate(ctx, env)
|
||||
}
|
||||
recordFastWriterStageDuration(registry, subject, "redis", statusFromError(err), time.Since(started))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
recordFastWriterRedisEnvelopeMetrics(registry, subject, result)
|
||||
recordFastWriterRedisFieldMetrics(registry, subject, result)
|
||||
recordFastWriterRedisE2EDuration(registry, subject, env)
|
||||
return nil
|
||||
}
|
||||
|
||||
func shouldFallbackFastBatchError(err error) bool {
|
||||
if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return false
|
||||
}
|
||||
return !isTransientFastBatchError(err)
|
||||
}
|
||||
|
||||
func isTransientFastBatchError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
text := strings.ToLower(strings.TrimSpace(err.Error()))
|
||||
return strings.Contains(text, "timeout") ||
|
||||
strings.Contains(text, "temporary") ||
|
||||
strings.Contains(text, "temporarily") ||
|
||||
strings.Contains(text, "connection refused") ||
|
||||
strings.Contains(text, "connection reset") ||
|
||||
strings.Contains(text, "connection closed") ||
|
||||
strings.Contains(text, "broken pipe") ||
|
||||
strings.Contains(text, "bad connection") ||
|
||||
strings.Contains(text, "i/o timeout") ||
|
||||
text == "eof" ||
|
||||
strings.Contains(text, "unexpected eof") ||
|
||||
strings.Contains(text, "server is down") ||
|
||||
strings.Contains(text, "network is unreachable") ||
|
||||
strings.Contains(text, "no route to host")
|
||||
}
|
||||
|
||||
func recordFastWriterConfigMetrics(registry *metrics.Registry, cfg config) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "workers"}, float64(cfg.Workers))
|
||||
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "batch_size"}, float64(cfg.BatchSize))
|
||||
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "fetch_wait_ms"}, float64(cfg.FetchWait.Milliseconds()))
|
||||
registry.SetGauge("vehicle_fast_writer_config", metrics.Labels{"setting": "operation_timeout_ms"}, float64(cfg.OperationWait.Milliseconds()))
|
||||
}
|
||||
|
||||
func boolGauge(value bool) float64 {
|
||||
if value {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
type fastSubjectGroup struct {
|
||||
subject string
|
||||
messages []*fastMessage
|
||||
envelopes []envelope.FrameEnvelope
|
||||
}
|
||||
|
||||
func fastSubjectGroups(messages []*fastMessage, envelopes []envelope.FrameEnvelope) []fastSubjectGroup {
|
||||
if len(envelopes) == 0 {
|
||||
return nil
|
||||
}
|
||||
fallback := fastBatchSubject(messages)
|
||||
groups := make([]fastSubjectGroup, 0, len(envelopes))
|
||||
indexBySubject := make(map[string]int, len(envelopes))
|
||||
for index, env := range envelopes {
|
||||
subject := fastMessageSubject(messages, index, fallback)
|
||||
groupIndex, exists := indexBySubject[subject]
|
||||
if !exists {
|
||||
groupIndex = len(groups)
|
||||
indexBySubject[subject] = groupIndex
|
||||
groups = append(groups, fastSubjectGroup{subject: subject})
|
||||
}
|
||||
groups[groupIndex].envelopes = append(groups[groupIndex].envelopes, env)
|
||||
if index >= 0 && index < len(messages) {
|
||||
groups[groupIndex].messages = append(groups[groupIndex].messages, messages[index])
|
||||
}
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func fastBatchSubject(messages []*fastMessage) string {
|
||||
if len(messages) == 0 {
|
||||
return "unknown"
|
||||
@@ -384,6 +737,16 @@ func fastBatchSubject(messages []*fastMessage) string {
|
||||
return subject
|
||||
}
|
||||
|
||||
func fastMessageSubject(messages []*fastMessage, index int, fallback string) string {
|
||||
if index >= 0 && index < len(messages) && strings.TrimSpace(messages[index].subject) != "" {
|
||||
return messages[index].subject
|
||||
}
|
||||
if strings.TrimSpace(fallback) != "" {
|
||||
return fallback
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func ensureStream(js nats.JetStreamContext, cfg config) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.StreamEnsureWait)
|
||||
defer cancel()
|
||||
@@ -416,15 +779,36 @@ func addFastMetric(registry *metrics.Registry, subject string, status string) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.IncCounter("vehicle_fast_writer_messages_total", metrics.Labels{"subject": subject, "status": status})
|
||||
labels := metrics.Labels{"subject": subject, "status": status}
|
||||
registry.IncCounter("vehicle_fast_writer_messages_total", labels)
|
||||
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_message_unix_seconds", labels)
|
||||
}
|
||||
|
||||
func setFastBatchPending(registry *metrics.Registry, messages int, envelopes int) {
|
||||
func addFastBatchFallbackMetric(registry *metrics.Registry, stage string, status string) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.SetGauge("vehicle_fast_writer_batch_pending_messages", nil, float64(messages))
|
||||
registry.SetGauge("vehicle_fast_writer_batch_pending_envelopes", nil, float64(envelopes))
|
||||
registry.IncCounter("vehicle_fast_writer_batch_fallback_total", metrics.Labels{
|
||||
"stage": stage,
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
|
||||
func addFastDecoupledUpdateMetric(registry *metrics.Registry, reason string, status string) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.IncCounter("vehicle_fast_writer_decoupled_updates_total", metrics.Labels{
|
||||
"reason": reason,
|
||||
"status": status,
|
||||
})
|
||||
}
|
||||
|
||||
func addFastBatchPending(registry *metrics.Registry, messages int, envelopes int) {
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
fastBatchPending.Add(registry, "vehicle_fast_writer_batch_pending_messages", "vehicle_fast_writer_batch_pending_envelopes", messages, envelopes)
|
||||
}
|
||||
|
||||
func recordFastNATSConsumerInfoMetrics(registry *metrics.Registry, cfg config, info *nats.ConsumerInfo) {
|
||||
@@ -441,11 +825,78 @@ func recordFastWriterStageDuration(registry *metrics.Registry, subject string, s
|
||||
if registry == nil {
|
||||
return
|
||||
}
|
||||
registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", metrics.Labels{
|
||||
labels := metrics.Labels{
|
||||
"subject": subject,
|
||||
"stage": stage,
|
||||
"status": status,
|
||||
}, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds()))
|
||||
}
|
||||
registry.ObserveHistogram("vehicle_fast_writer_stage_duration_ms_histogram", labels, fastWriterStageDurationBucketsMS, float64(elapsed.Milliseconds()))
|
||||
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_stage_unix_seconds", labels)
|
||||
}
|
||||
|
||||
func recordFastWriterRedisE2EDurationMessages(registry *metrics.Registry, messages []*fastMessage, envelopes []envelope.FrameEnvelope, fallback string) {
|
||||
for index, env := range envelopes {
|
||||
recordFastWriterRedisE2EDuration(registry, fastMessageSubject(messages, index, fallback), env)
|
||||
}
|
||||
}
|
||||
|
||||
func recordFastWriterRedisE2EDuration(registry *metrics.Registry, subject string, env envelope.FrameEnvelope) {
|
||||
if registry == nil || env.ReceivedAtMS <= 0 {
|
||||
return
|
||||
}
|
||||
elapsedMS := time.Since(time.UnixMilli(env.ReceivedAtMS)).Milliseconds()
|
||||
if elapsedMS < 0 {
|
||||
elapsedMS = 0
|
||||
}
|
||||
labels := metrics.Labels{
|
||||
"subject": subject,
|
||||
}
|
||||
registry.ObserveHistogram("vehicle_fast_writer_redis_e2e_duration_ms_histogram", labels, fastWriterRedisE2EDurationBucketsMS, float64(elapsedMS))
|
||||
p99, samples := fastWriterRedisE2ERecent.Observe(subject, float64(elapsedMS))
|
||||
registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_p99_ms", labels, p99)
|
||||
registry.SetGauge("vehicle_fast_writer_redis_e2e_recent_samples", labels, float64(samples))
|
||||
metrics.RecordLastActivity(registry, "vehicle_fast_writer_last_redis_e2e_unix_seconds", labels)
|
||||
}
|
||||
|
||||
func recordFastWriterRedisFieldMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) {
|
||||
if registry == nil || result.FieldsSeen == 0 {
|
||||
return
|
||||
}
|
||||
addFastWriterRedisFieldMetric(registry, subject, "seen", result.FieldsSeen)
|
||||
addFastWriterRedisFieldMetric(registry, subject, "written", result.FieldsWritten)
|
||||
addFastWriterRedisFieldMetric(registry, subject, "skipped_stale", result.FieldsSkippedStale)
|
||||
}
|
||||
|
||||
func recordFastWriterRedisEnvelopeMetrics(registry *metrics.Registry, subject string, result realtime.FastUpdateResult) {
|
||||
if registry == nil || result.EnvelopesSeen == 0 {
|
||||
return
|
||||
}
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "seen", result.EnvelopesSeen)
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "updated", result.EnvelopesUpdated)
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_non_realtime", result.EnvelopesSkippedNonRealtime)
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vin", result.EnvelopesSkippedMissingVIN)
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_vehicle_key", result.EnvelopesSkippedMissingVehicleKey)
|
||||
addFastWriterRedisEnvelopeMetric(registry, subject, "skipped_missing_fields", result.EnvelopesSkippedMissingFields)
|
||||
}
|
||||
|
||||
func addFastWriterRedisEnvelopeMetric(registry *metrics.Registry, subject string, status string, value int) {
|
||||
if value <= 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_fast_writer_redis_envelopes_total", metrics.Labels{
|
||||
"subject": subject,
|
||||
"status": status,
|
||||
}, float64(value))
|
||||
}
|
||||
|
||||
func addFastWriterRedisFieldMetric(registry *metrics.Registry, subject string, status string, value int) {
|
||||
if value <= 0 {
|
||||
return
|
||||
}
|
||||
registry.AddCounter("vehicle_fast_writer_redis_fields_total", metrics.Labels{
|
||||
"subject": subject,
|
||||
"status": status,
|
||||
}, float64(value))
|
||||
}
|
||||
|
||||
func statusFromError(err error) string {
|
||||
@@ -487,6 +938,20 @@ func envInt64(key string, fallback int64) int64 {
|
||||
return parsed
|
||||
}
|
||||
|
||||
func envBool(key string, fallback bool) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(os.Getenv(key)))
|
||||
switch value {
|
||||
case "":
|
||||
return fallback
|
||||
case "1", "true", "yes", "y", "on":
|
||||
return true
|
||||
case "0", "false", "no", "n", "off":
|
||||
return false
|
||||
default:
|
||||
return fallback
|
||||
}
|
||||
}
|
||||
|
||||
func splitCSV(value string) []string {
|
||||
var out []string
|
||||
for _, item := range strings.Split(value, ",") {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||
)
|
||||
|
||||
func TestProcessFastMessageWritesTDengineAndRedisBeforeAck(t *testing.T) {
|
||||
@@ -58,8 +59,45 @@ func TestProcessFastMessageDoesNotAckWhenRedisFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageUpdatesRedisButDoesNotAckOnTransientTDengineFailure(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-td-transient-single"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{err: errors.New("connection reset by peer")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, appender, updater, msg); err == nil {
|
||||
t.Fatal("processFastMessage() error = nil, want tdengine transient failure")
|
||||
}
|
||||
if updater.count != 1 || ackCount != 0 {
|
||||
t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("tdengine transient redis catchup metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("message should not be counted ok because it was not acked:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3"}
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3", ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli()}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -77,6 +115,14 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_count{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_redis_e2e_unix_seconds{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer stage metric missing %s:\n%s", want, text)
|
||||
@@ -84,6 +130,151 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastWriterRedisE2EDurationSkipsMissingReceiveTime(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
recordFastWriterRedisE2EDuration(registry, "vehicle.raw.go.jt808.v1", envelope.FrameEnvelope{})
|
||||
|
||||
if text := registry.Render(); strings.Contains(text, "vehicle_fast_writer_redis_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_fast_writer_redis_e2e_recent") {
|
||||
t.Fatalf("e2e metric should be skipped when received_at_ms is missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageAcksInvalidJSONAndRecordsMetric(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
updater := &recordingFastUpdater{}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count = %d, want 1", ackCount)
|
||||
}
|
||||
if updater.count != 0 || updater.batchCount != 0 {
|
||||
t.Fatalf("updater counts = single %d batch %d, want 0", updater.count, updater.batchCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("invalid json metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("invalid json should not be counted as ok:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageAcksExplicitNonRawEventKindWithoutWriting(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
MessageID: "0x0200",
|
||||
VIN: "VIN001",
|
||||
}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{}
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, appender, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if appender.count != 0 || updater.count != 0 || updater.batchCount != 0 {
|
||||
t.Fatalf("appends=%d updates=%d batch_updates=%d, want no writes", appender.count, updater.count, updater.batchCount)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count=%d, want 1", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_count{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("mismatch metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("mismatched event should not be counted ok:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageRecordsRedisFieldMetrics(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fields"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 4,
|
||||
EnvelopesUpdated: 1,
|
||||
EnvelopesSkippedNonRealtime: 1,
|
||||
EnvelopesSkippedMissingVIN: 1,
|
||||
EnvelopesSkippedMissingVehicleKey: 0,
|
||||
EnvelopesSkippedMissingFields: 1,
|
||||
FieldsSeen: 3,
|
||||
FieldsWritten: 2,
|
||||
FieldsSkippedStale: 1,
|
||||
}}
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if updater.resultCount != 1 {
|
||||
t.Fatalf("FastUpdateWithResult count=%d, want 1", updater.resultCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 4`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_vin",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_non_realtime",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_fields",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 3`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddFastMetricRecordsLastMessage(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
addFastMetric(registry, "vehicle.raw.go.gb32960.v1", "ok")
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_last_message_unix_seconds{status="ok",subject="vehicle.raw.go.gb32960.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer message metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-4"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-5"}
|
||||
@@ -149,6 +340,407 @@ func TestProcessFastBatchUsesRedisBatchUpdaterWhenAvailable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallsBackToSinglesAfterNonTransientTDengineBatchFailure(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-td-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{batchErr: errors.New("syntax error in batch insert")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 2 {
|
||||
t.Fatalf("batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 2 || ackCount != 2 {
|
||||
t.Fatalf("updates=%d acks=%d, want 2/2", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="ok"} 1`) {
|
||||
t.Fatalf("tdengine fallback metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchUpdatesRedisButDoesNotAckOnTransientTDengineBatchFailure(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-transient"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{batchErr: errors.New("i/o timeout")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil {
|
||||
t.Fatal("processFastBatch() error = nil, want transient tdengine failure")
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 0 {
|
||||
t.Fatalf("batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 1 || ackCount != 0 {
|
||||
t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="skipped_transient"} 1`,
|
||||
`vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("tdengine transient catchup metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallsBackToRedisSinglesAfterNonTransientRedisBatchFailure(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-redis-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-redis-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{}
|
||||
updater := &recordingFastUpdater{batchErr: errors.New("WRONGTYPE operation against a key holding the wrong kind of value")}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 0 {
|
||||
t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.count != 2 {
|
||||
t.Fatalf("redis batch updates=%d single updates=%d, want 1/2", updater.batchCount, updater.count)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count=%d, want 2", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="redis",status="ok"} 1`) {
|
||||
t.Fatalf("redis fallback metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallbackLeavesFailedAndRemainingMessagesUnacked(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-prefix-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-prefix-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{
|
||||
batchErr: errors.New("syntax error in batch insert"),
|
||||
singleErr: errors.New("single insert failed"),
|
||||
failSingleOnCount: 2,
|
||||
}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil {
|
||||
t.Fatal("processFastBatch() error = nil, want fallback single write failure")
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 2 {
|
||||
t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 1 || ackCount != 1 {
|
||||
t.Fatalf("updates=%d acks=%d, want successful prefix 1/1", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 1`) {
|
||||
t.Fatalf("tdengine fallback error metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisFieldMetrics(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-fields-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-fields-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 2,
|
||||
EnvelopesUpdated: 2,
|
||||
FieldsSeen: 10,
|
||||
FieldsWritten: 7,
|
||||
FieldsSkippedStale: 3,
|
||||
}}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchResultCount != 1 || updater.batchCount != 0 {
|
||||
t.Fatalf("batch result count=%d legacy batch count=%d, want 1/0", updater.batchResultCount, updater.batchCount)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count=%d, want 2", ackCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 10`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 3`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 7`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisE2EByOriginalSubject(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventID: "evt-batch-e2e-1",
|
||||
ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli(),
|
||||
}
|
||||
second := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "VIN002",
|
||||
EventID: "evt-batch-e2e-2",
|
||||
ReceivedAtMS: time.Now().Add(-25 * time.Millisecond).UnixMilli(),
|
||||
}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 2,
|
||||
EnvelopesUpdated: 2,
|
||||
}}
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload},
|
||||
{subject: "vehicle.raw.go.gb32960.v1", data: secondPayload},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis e2e metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="mixed"}`) {
|
||||
t.Fatalf("redis e2e metric should keep original subjects, not mixed:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisFieldMetricsByOriginalSubjectForMixedBatch(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-mixed-fields-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", EventID: "evt-batch-mixed-fields-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastDynamicResultUpdater{}
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload},
|
||||
{subject: "vehicle.raw.go.gb32960.v1", data: secondPayload},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchResultCount != 2 {
|
||||
t.Fatalf("batch result count=%d, want one batch per subject", updater.batchResultCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `subject="mixed"`) {
|
||||
t.Fatalf("mixed subject should not be used for per-subject Redis metrics:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchAcksInvalidJSONWithoutMarkingItOK(t *testing.T) {
|
||||
valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-valid"}
|
||||
validPayload, err := valid.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count = %d, want 2", ackCount)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want one valid row", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("batch invalid json metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchSkipsExplicitNonRawEventKindAndWritesValidRows(t *testing.T) {
|
||||
mismatched := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindFields, VIN: "VIN_BAD", EventID: "evt-fields-on-raw"}
|
||||
valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindRaw, VIN: "VIN001", EventID: "evt-valid-raw"}
|
||||
mismatchedPayload, err := mismatched.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validPayload, err := valid.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: mismatchedPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count = %d, want 2", ackCount)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want only valid row", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("batch mismatch metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchCanSkipTDengineAppender(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-no-td"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want 1/1", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count=%d, want 1", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if strings.Contains(text, `stage="tdengine"`) {
|
||||
t.Fatalf("tdengine stage should not be recorded when appender is nil:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, `stage="redis"`) || !strings.Contains(text, `stage="ack"`) {
|
||||
t.Fatalf("redis and ack stages should still be recorded:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"}
|
||||
@@ -208,6 +800,77 @@ func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchPendingAggregatesConcurrentWorkers(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fast-pending-concurrent-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fast-pending-concurrent-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
firstStarted := make(chan struct{})
|
||||
secondStarted := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
appenderFor := func(started chan struct{}) *recordingFastAppender {
|
||||
return &recordingFastAppender{
|
||||
onAppendBatch: func() {
|
||||
close(started)
|
||||
<-release
|
||||
},
|
||||
}
|
||||
}
|
||||
messages := func(payload []byte, count int) []*fastMessage {
|
||||
out := make([]*fastMessage, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
out = append(out, &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }})
|
||||
}
|
||||
return out
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
var firstErr error
|
||||
var secondErr error
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
firstErr = processFastBatch(context.Background(), registry, appenderFor(firstStarted), &recordingFastUpdater{}, messages(firstPayload, 2))
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
secondErr = processFastBatch(context.Background(), registry, appenderFor(secondStarted), &recordingFastUpdater{}, messages(secondPayload, 3))
|
||||
}()
|
||||
<-firstStarted
|
||||
<-secondStarted
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_batch_pending_messages 5`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 5`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("aggregate pending metric missing %s during concurrent append:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if firstErr != nil || secondErr != nil {
|
||||
t.Fatalf("processFastBatch errors = %v / %v", firstErr, secondErr)
|
||||
}
|
||||
text = registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("aggregate pending metric should reset after concurrent append, missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"}
|
||||
@@ -230,6 +893,41 @@ func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFastWorkerShutdownFetchError(t *testing.T) {
|
||||
if isFastWorkerShutdownFetchError(context.Background(), errors.New("temporary nats failure")) {
|
||||
t.Fatal("temporary failure should not be treated as shutdown")
|
||||
}
|
||||
if !isFastWorkerShutdownFetchError(context.Background(), nats.ErrConnectionClosed) {
|
||||
t.Fatal("connection closed should stop worker without noisy error log")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if !isFastWorkerShutdownFetchError(ctx, errors.New("any fetch error after cancellation")) {
|
||||
t.Fatal("cancelled context should stop worker without noisy error log")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTransientFastFetchError(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("nats: disconnected during fetch"),
|
||||
errors.New("nats: connection closed"),
|
||||
errors.New("read tcp: connection reset by peer"),
|
||||
errors.New("write tcp: broken pipe"),
|
||||
errors.New("temporary network unavailable"),
|
||||
errors.New("i/o timeout"),
|
||||
} {
|
||||
if !isTransientFastFetchError(err) {
|
||||
t.Fatalf("isTransientFastFetchError(%q) = false, want true", err.Error())
|
||||
}
|
||||
}
|
||||
if isTransientFastFetchError(errors.New("permission denied")) {
|
||||
t.Fatal("permission denied should remain an unexpected fetch error")
|
||||
}
|
||||
if isTransientFastFetchError(nil) {
|
||||
t.Fatal("nil should not be treated as transient")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) {
|
||||
t.Setenv("FAST_WRITER_WORKERS", "12")
|
||||
t.Setenv("FAST_WRITER_TDENGINE_MAX_OPEN_CONNS", "")
|
||||
@@ -244,6 +942,28 @@ func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsTDengineStageDisabled(t *testing.T) {
|
||||
cfg := loadConfig()
|
||||
|
||||
if cfg.TDengineEnabled {
|
||||
t.Fatal("TDengineEnabled = true, want production default false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigCanEnableTDengineStageExplicitly(t *testing.T) {
|
||||
for _, value := range []string{"true", "1", "yes", "on"} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
t.Setenv("FAST_WRITER_TDENGINE_ENABLED", value)
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
if !cfg.TDengineEnabled {
|
||||
t.Fatalf("TDengineEnabled = false for %q, want explicit true", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) {
|
||||
cfg := loadConfig()
|
||||
|
||||
@@ -256,12 +976,16 @@ func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) {
|
||||
if got, want := cfg.OperationWait, time.Second; got != want {
|
||||
t.Fatalf("OperationWait = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := cfg.FetchWait, 20*time.Millisecond; got != want {
|
||||
t.Fatalf("FetchWait = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) {
|
||||
t.Setenv("NATS_STREAM_MAX_BYTES", "1073741824")
|
||||
t.Setenv("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", "90")
|
||||
t.Setenv("FAST_WRITER_OPERATION_TIMEOUT_MS", "1500")
|
||||
t.Setenv("FAST_WRITER_FETCH_WAIT_MS", "45")
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
@@ -274,6 +998,32 @@ func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) {
|
||||
if got, want := cfg.OperationWait, 1500*time.Millisecond; got != want {
|
||||
t.Fatalf("OperationWait = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := cfg.FetchWait, 45*time.Millisecond; got != want {
|
||||
t.Fatalf("FetchWait = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastWriterConfigMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
recordFastWriterConfigMetrics(registry, config{
|
||||
BatchSize: 120,
|
||||
FetchWait: 20 * time.Millisecond,
|
||||
OperationWait: 1500 * time.Millisecond,
|
||||
Workers: 8,
|
||||
})
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_config{setting="batch_size"} 120`,
|
||||
`vehicle_fast_writer_config{setting="fetch_wait_ms"} 20`,
|
||||
`vehicle_fast_writer_config{setting="operation_timeout_ms"} 1500`,
|
||||
`vehicle_fast_writer_config{setting="workers"} 8`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer config metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) {
|
||||
@@ -292,15 +1042,30 @@ func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) {
|
||||
}
|
||||
|
||||
type recordingFastAppender struct {
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
onAppendBatch func()
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
failSingleOnCount int
|
||||
onAppendBatch func()
|
||||
}
|
||||
|
||||
func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error {
|
||||
a.count++
|
||||
if a.failSingleOnCount > 0 && a.count == a.failSingleOnCount {
|
||||
if a.singleErr != nil {
|
||||
return a.singleErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
if a.failSingleOnCount > 0 {
|
||||
return nil
|
||||
}
|
||||
if a.singleErr != nil {
|
||||
return a.singleErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
|
||||
@@ -310,24 +1075,45 @@ func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelop
|
||||
if a.onAppendBatch != nil {
|
||||
a.onAppendBatch()
|
||||
}
|
||||
if a.batchErr != nil {
|
||||
return a.batchErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
|
||||
type recordingFastUpdater struct {
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
failSingleOnCount int
|
||||
}
|
||||
|
||||
func (u *recordingFastUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
u.count++
|
||||
if u.failSingleOnCount > 0 && u.count == u.failSingleOnCount {
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
if u.failSingleOnCount > 0 {
|
||||
return nil
|
||||
}
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
||||
u.batchCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.batchErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
@@ -340,3 +1126,73 @@ func (u *recordingFastSingleUpdater) FastUpdate(context.Context, envelope.FrameE
|
||||
u.count++
|
||||
return u.err
|
||||
}
|
||||
|
||||
type recordingFastResultUpdater struct {
|
||||
count int
|
||||
resultCount int
|
||||
batchCount int
|
||||
batchResultCount int
|
||||
batchRows int
|
||||
result realtime.FastUpdateResult
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
u.count++
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.resultCount++
|
||||
if u.singleErr != nil {
|
||||
return u.result, u.singleErr
|
||||
}
|
||||
return u.result, u.err
|
||||
}
|
||||
|
||||
type recordingFastDynamicResultUpdater struct {
|
||||
batchResultCount int
|
||||
batchRows int
|
||||
err error
|
||||
}
|
||||
|
||||
func (u *recordingFastDynamicResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastDynamicResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.batchResultCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.err != nil {
|
||||
return realtime.FastUpdateResult{}, u.err
|
||||
}
|
||||
return realtime.FastUpdateResult{
|
||||
EnvelopesSeen: len(envs),
|
||||
EnvelopesUpdated: len(envs),
|
||||
FieldsSeen: len(envs),
|
||||
FieldsWritten: len(envs),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
||||
u.batchCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.batchErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.batchResultCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.result, u.batchErr
|
||||
}
|
||||
return u.result, u.err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user