package main import ( "context" "errors" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime" ) func TestProcessFastMessageWritesTDengineAndRedisBeforeAck(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", EventID: "evt-1"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } appender := &recordingFastAppender{} updater := &recordingFastSingleUpdater{} ackCount := 0 msg := &fastMessage{data: payload, ack: func() error { ackCount++ return nil }} if err := processFastMessage(context.Background(), nil, appender, updater, msg); err != nil { t.Fatalf("processFastMessage() error = %v", err) } if appender.count != 1 || updater.count != 1 || ackCount != 1 { t.Fatalf("appends=%d updates=%d acks=%d", appender.count, updater.count, ackCount) } } func TestProcessFastMessageDoesNotAckWhenRedisFails(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-2"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } appender := &recordingFastAppender{} updater := &recordingFastUpdater{err: errors.New("redis down")} ackCount := 0 msg := &fastMessage{data: payload, ack: func() error { ackCount++ return nil }} if err := processFastMessage(context.Background(), nil, appender, updater, msg); err == nil { t.Fatal("processFastMessage() error = nil, want redis failure") } if ackCount != 0 { t.Fatalf("ack count = %d, want 0", ackCount) } } func TestProcessFastMessageUpdatesRedisButDoesNotAckOnTransientTDengineFailure(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-td-transient-single"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{err: errors.New("connection reset by peer")} updater := &recordingFastSingleUpdater{} ackCount := 0 msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++ return nil }} if err := processFastMessage(context.Background(), registry, appender, updater, msg); err == nil { t.Fatal("processFastMessage() error = nil, want tdengine transient failure") } if updater.count != 1 || ackCount != 0 { t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("tdengine transient redis catchup metric missing %s:\n%s", want, text) } } if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { t.Fatalf("message should not be counted ok because it was not acked:\n%s", text) } } func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3", ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli()} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }} if err := processFastMessage(context.Background(), registry, &recordingFastAppender{}, &recordingFastUpdater{}, msg); err != nil { t.Fatalf("processFastMessage() error = %v", err) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_count{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_last_stage_unix_seconds{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_last_stage_unix_seconds{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_last_redis_e2e_unix_seconds{subject="vehicle.raw.go.jt808.v1"} `, } { if !strings.Contains(text, want) { t.Fatalf("fast writer stage metric missing %s:\n%s", want, text) } } } func TestRecordFastWriterRedisE2EDurationSkipsMissingReceiveTime(t *testing.T) { registry := metrics.NewRegistry() recordFastWriterRedisE2EDuration(registry, "vehicle.raw.go.jt808.v1", envelope.FrameEnvelope{}) if text := registry.Render(); strings.Contains(text, "vehicle_fast_writer_redis_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_fast_writer_redis_e2e_recent") { t.Fatalf("e2e metric should be skipped when received_at_ms is missing:\n%s", text) } } func TestProcessFastMessageAcksInvalidJSONAndRecordsMetric(t *testing.T) { registry := metrics.NewRegistry() ackCount := 0 msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { ackCount++ return nil }} updater := &recordingFastUpdater{} if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil { t.Fatalf("processFastMessage() error = %v", err) } if ackCount != 1 { t.Fatalf("ack count = %d, want 1", ackCount) } if updater.count != 0 || updater.batchCount != 0 { t.Fatalf("updater counts = single %d batch %d, want 0", updater.count, updater.batchCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("invalid json metric missing %s:\n%s", want, text) } } if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { t.Fatalf("invalid json should not be counted as ok:\n%s", text) } } func TestProcessFastMessageAcksExplicitNonRawEventKindWithoutWriting(t *testing.T) { env := envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindFields, MessageID: "0x0200", VIN: "VIN001", } payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{} updater := &recordingFastUpdater{} ackCount := 0 msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++ return nil }} if err := processFastMessage(context.Background(), registry, appender, updater, msg); err != nil { t.Fatalf("processFastMessage() error = %v", err) } if appender.count != 0 || updater.count != 0 || updater.batchCount != 0 { t.Fatalf("appends=%d updates=%d batch_updates=%d, want no writes", appender.count, updater.count, updater.batchCount) } if ackCount != 1 { t.Fatalf("ack count=%d, want 1", ackCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_count{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("mismatch metric missing %s:\n%s", want, text) } } if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) { t.Fatalf("mismatched event should not be counted ok:\n%s", text) } } func TestProcessFastMessageRecordsRedisFieldMetrics(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fields"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ EnvelopesSeen: 4, EnvelopesUpdated: 1, EnvelopesSkippedNonRealtime: 1, EnvelopesSkippedMissingVIN: 1, EnvelopesSkippedMissingVehicleKey: 0, EnvelopesSkippedMissingFields: 1, FieldsSeen: 3, FieldsWritten: 2, FieldsSkippedStale: 1, }} msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }} if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil { t.Fatalf("processFastMessage() error = %v", err) } if updater.resultCount != 1 { t.Fatalf("FastUpdateWithResult count=%d, want 1", updater.resultCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 4`, `vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_vin",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_envelopes_total{status="skipped_non_realtime",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_fields",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 3`, `vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 2`, } { if !strings.Contains(text, want) { t.Fatalf("redis field metric missing %s:\n%s", want, text) } } } func TestAddFastMetricRecordsLastMessage(t *testing.T) { registry := metrics.NewRegistry() addFastMetric(registry, "vehicle.raw.go.gb32960.v1", "ok") text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 1`, `vehicle_fast_writer_last_message_unix_seconds{status="ok",subject="vehicle.raw.go.gb32960.v1"} `, } { if !strings.Contains(text, want) { t.Fatalf("fast writer message metric missing %s:\n%s", want, text) } } } func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-4"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-5"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } appender := &recordingFastAppender{} updater := &recordingFastSingleUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), nil, appender, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if appender.count != 0 { t.Fatalf("AppendAll count = %d, want 0", appender.count) } if appender.batchCount != 1 || appender.batchRows != 2 { t.Fatalf("AppendAllBatch count=%d rows=%d, want count=1 rows=2", appender.batchCount, appender.batchRows) } if updater.count != 2 || ackCount != 2 { t.Fatalf("updates=%d acks=%d, want 2/2", updater.count, ackCount) } } func TestProcessFastBatchUsesRedisBatchUpdaterWhenAvailable(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } updater := &recordingFastUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), nil, &recordingFastAppender{}, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if updater.batchCount != 1 || updater.batchRows != 2 { t.Fatalf("FastUpdateBatch count=%d rows=%d, want count=1 rows=2", updater.batchCount, updater.batchRows) } if updater.count != 0 { t.Fatalf("FastUpdate count=%d, want 0 when batch updater is available", updater.count) } if ackCount != 2 { t.Fatalf("ack count=%d, want 2", ackCount) } } func TestProcessFastBatchFallsBackToSinglesAfterNonTransientTDengineBatchFailure(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-td-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{batchErr: errors.New("syntax error in batch insert")} updater := &recordingFastSingleUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if appender.batchCount != 1 || appender.count != 2 { t.Fatalf("batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count) } if updater.count != 2 || ackCount != 2 { t.Fatalf("updates=%d acks=%d, want 2/2", updater.count, ackCount) } text := registry.Render() if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="ok"} 1`) { t.Fatalf("tdengine fallback metric missing:\n%s", text) } } func TestProcessFastBatchUpdatesRedisButDoesNotAckOnTransientTDengineBatchFailure(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-transient"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{batchErr: errors.New("i/o timeout")} updater := &recordingFastSingleUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil { t.Fatal("processFastBatch() error = nil, want transient tdengine failure") } if appender.batchCount != 1 || appender.count != 0 { t.Fatalf("batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count) } if updater.count != 1 || ackCount != 0 { t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="skipped_transient"} 1`, `vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`, `vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("tdengine transient catchup metric missing %s:\n%s", want, text) } } } func TestProcessFastBatchFallsBackToRedisSinglesAfterNonTransientRedisBatchFailure(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-redis-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-redis-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{} updater := &recordingFastUpdater{batchErr: errors.New("WRONGTYPE operation against a key holding the wrong kind of value")} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if appender.batchCount != 1 || appender.count != 0 { t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count) } if updater.batchCount != 1 || updater.count != 2 { t.Fatalf("redis batch updates=%d single updates=%d, want 1/2", updater.batchCount, updater.count) } if ackCount != 2 { t.Fatalf("ack count=%d, want 2", ackCount) } text := registry.Render() if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="redis",status="ok"} 1`) { t.Fatalf("redis fallback metric missing:\n%s", text) } } func TestProcessFastBatchFallbackLeavesFailedAndRemainingMessagesUnacked(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-prefix-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-prefix-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appender := &recordingFastAppender{ batchErr: errors.New("syntax error in batch insert"), singleErr: errors.New("single insert failed"), failSingleOnCount: 2, } updater := &recordingFastSingleUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil { t.Fatal("processFastBatch() error = nil, want fallback single write failure") } if appender.batchCount != 1 || appender.count != 2 { t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count) } if updater.count != 1 || ackCount != 1 { t.Fatalf("updates=%d acks=%d, want successful prefix 1/1", updater.count, ackCount) } text := registry.Render() if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 1`) { t.Fatalf("tdengine fallback error metric missing:\n%s", text) } } func TestProcessFastBatchRecordsRedisFieldMetrics(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-fields-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-fields-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ EnvelopesSeen: 2, EnvelopesUpdated: 2, FieldsSeen: 10, FieldsWritten: 7, FieldsSkippedStale: 3, }} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if updater.batchResultCount != 1 || updater.batchCount != 0 { t.Fatalf("batch result count=%d legacy batch count=%d, want 1/0", updater.batchResultCount, updater.batchCount) } if ackCount != 2 { t.Fatalf("ack count=%d, want 2", ackCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 2`, `vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 2`, `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 10`, `vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 3`, `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 7`, } { if !strings.Contains(text, want) { t.Fatalf("redis field metric missing %s:\n%s", want, text) } } } func TestProcessFastBatchRecordsRedisE2EByOriginalSubject(t *testing.T) { first := envelope.FrameEnvelope{ Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-e2e-1", ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli(), } second := envelope.FrameEnvelope{ Protocol: envelope.ProtocolGB32960, VIN: "VIN002", EventID: "evt-batch-e2e-2", ReceivedAtMS: time.Now().Add(-25 * time.Millisecond).UnixMilli(), } firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{ EnvelopesSeen: 2, EnvelopesUpdated: 2, }} msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload}, {subject: "vehicle.raw.go.gb32960.v1", data: secondPayload}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.gb32960.v1"} 1`, `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `, `vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} `, } { if !strings.Contains(text, want) { t.Fatalf("redis e2e metric missing %s:\n%s", want, text) } } if strings.Contains(text, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="mixed"}`) { t.Fatalf("redis e2e metric should keep original subjects, not mixed:\n%s", text) } } func TestProcessFastBatchRecordsRedisFieldMetricsByOriginalSubjectForMixedBatch(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-mixed-fields-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", EventID: "evt-batch-mixed-fields-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastDynamicResultUpdater{} msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload}, {subject: "vehicle.raw.go.gb32960.v1", data: secondPayload}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if updater.batchResultCount != 2 { t.Fatalf("batch result count=%d, want one batch per subject", updater.batchResultCount) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.gb32960.v1"} 1`, } { if !strings.Contains(text, want) { t.Fatalf("redis field metric missing %s:\n%s", want, text) } } if strings.Contains(text, `subject="mixed"`) { t.Fatalf("mixed subject should not be used for per-subject Redis metrics:\n%s", text) } } func TestProcessFastBatchAcksInvalidJSONWithoutMarkingItOK(t *testing.T) { valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-valid"} validPayload, err := valid.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if ackCount != 2 { t.Fatalf("ack count = %d, want 2", ackCount) } if updater.batchCount != 1 || updater.batchRows != 1 { t.Fatalf("redis batch count=%d rows=%d, want one valid row", updater.batchCount, updater.batchRows) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`, `vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_batch_pending_messages 0`, `vehicle_fast_writer_batch_pending_envelopes 0`, } { if !strings.Contains(text, want) { t.Fatalf("batch invalid json metric missing %s:\n%s", want, text) } } } func TestProcessFastBatchSkipsExplicitNonRawEventKindAndWritesValidRows(t *testing.T) { mismatched := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindFields, VIN: "VIN_BAD", EventID: "evt-fields-on-raw"} valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindRaw, VIN: "VIN001", EventID: "evt-valid-raw"} mismatchedPayload, err := mismatched.MarshalJSONBytes() if err != nil { t.Fatal(err) } validPayload, err := valid.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: mismatchedPayload, ack: func() error { ackCount++; return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if ackCount != 2 { t.Fatalf("ack count = %d, want 2", ackCount) } if updater.batchCount != 1 || updater.batchRows != 1 { t.Fatalf("redis batch count=%d rows=%d, want only valid row", updater.batchCount, updater.batchRows) } text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`, `vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`, `vehicle_fast_writer_batch_pending_messages 0`, `vehicle_fast_writer_batch_pending_envelopes 0`, } { if !strings.Contains(text, want) { t.Fatalf("batch mismatch metric missing %s:\n%s", want, text) } } } func TestProcessFastBatchCanSkipTDengineAppender(t *testing.T) { env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-no-td"} payload, err := env.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() updater := &recordingFastUpdater{} ackCount := 0 msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }}, } if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil { t.Fatalf("processFastBatch() error = %v", err) } if updater.batchCount != 1 || updater.batchRows != 1 { t.Fatalf("redis batch count=%d rows=%d, want 1/1", updater.batchCount, updater.batchRows) } if ackCount != 1 { t.Fatalf("ack count=%d, want 1", ackCount) } text := registry.Render() if strings.Contains(text, `stage="tdengine"`) { t.Fatalf("tdengine stage should not be recorded when appender is nil:\n%s", text) } if !strings.Contains(text, `stage="redis"`) || !strings.Contains(text, `stage="ack"`) { t.Fatalf("redis and ack stages should still be recorded:\n%s", text) } } func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() appenderStarted := make(chan struct{}) appenderRelease := make(chan struct{}) appender := &recordingFastAppender{ onAppendBatch: func() { close(appenderStarted) <-appenderRelease }, } msgs := []*fastMessage{ {subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { return nil }}, {subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { return nil }}, } var wg sync.WaitGroup wg.Add(1) var processErr error go func() { defer wg.Done() processErr = processFastBatch(context.Background(), registry, appender, &recordingFastUpdater{}, msgs) }() <-appenderStarted text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_batch_pending_messages 2`, `vehicle_fast_writer_batch_pending_envelopes 2`, } { if !strings.Contains(text, want) { t.Fatalf("pending metric missing %s while appending:\n%s", want, text) } } close(appenderRelease) wg.Wait() if processErr != nil { t.Fatalf("processFastBatch() error = %v", processErr) } text = registry.Render() for _, want := range []string{ `vehicle_fast_writer_batch_pending_messages 0`, `vehicle_fast_writer_batch_pending_envelopes 0`, } { if !strings.Contains(text, want) { t.Fatalf("pending metric should reset after append, missing %s:\n%s", want, text) } } } func TestProcessFastBatchPendingAggregatesConcurrentWorkers(t *testing.T) { first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fast-pending-concurrent-1"} second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fast-pending-concurrent-2"} firstPayload, err := first.MarshalJSONBytes() if err != nil { t.Fatal(err) } secondPayload, err := second.MarshalJSONBytes() if err != nil { t.Fatal(err) } registry := metrics.NewRegistry() firstStarted := make(chan struct{}) secondStarted := make(chan struct{}) release := make(chan struct{}) appenderFor := func(started chan struct{}) *recordingFastAppender { return &recordingFastAppender{ onAppendBatch: func() { close(started) <-release }, } } messages := func(payload []byte, count int) []*fastMessage { out := make([]*fastMessage, 0, count) for i := 0; i < count; i++ { out = append(out, &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }}) } return out } var wg sync.WaitGroup wg.Add(2) var firstErr error var secondErr error go func() { defer wg.Done() firstErr = processFastBatch(context.Background(), registry, appenderFor(firstStarted), &recordingFastUpdater{}, messages(firstPayload, 2)) }() go func() { defer wg.Done() secondErr = processFastBatch(context.Background(), registry, appenderFor(secondStarted), &recordingFastUpdater{}, messages(secondPayload, 3)) }() <-firstStarted <-secondStarted text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_batch_pending_messages 5`, `vehicle_fast_writer_batch_pending_envelopes 5`, } { if !strings.Contains(text, want) { t.Fatalf("aggregate pending metric missing %s during concurrent append:\n%s", want, text) } } close(release) wg.Wait() if firstErr != nil || secondErr != nil { t.Fatalf("processFastBatch errors = %v / %v", firstErr, secondErr) } text = registry.Render() for _, want := range []string{ `vehicle_fast_writer_batch_pending_messages 0`, `vehicle_fast_writer_batch_pending_envelopes 0`, } { if !strings.Contains(text, want) { t.Fatalf("aggregate pending metric should reset after concurrent append, missing %s:\n%s", want, text) } } } func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) { registry := metrics.NewRegistry() cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"} recordFastNATSConsumerInfoMetrics(registry, cfg, &nats.ConsumerInfo{ NumPending: 23, NumAckPending: 5, NumWaiting: 3, }) text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_nats_consumer_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 23`, `vehicle_fast_writer_nats_consumer_ack_pending{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 5`, `vehicle_fast_writer_nats_consumer_waiting{consumer="vehicle-fast-writer",stream="VEHICLE_INGEST"} 3`, } { if !strings.Contains(text, want) { t.Fatalf("metrics missing %s:\n%s", want, text) } } } func TestIsFastWorkerShutdownFetchError(t *testing.T) { if isFastWorkerShutdownFetchError(context.Background(), errors.New("temporary nats failure")) { t.Fatal("temporary failure should not be treated as shutdown") } if !isFastWorkerShutdownFetchError(context.Background(), nats.ErrConnectionClosed) { t.Fatal("connection closed should stop worker without noisy error log") } ctx, cancel := context.WithCancel(context.Background()) cancel() if !isFastWorkerShutdownFetchError(ctx, errors.New("any fetch error after cancellation")) { t.Fatal("cancelled context should stop worker without noisy error log") } } func TestIsTransientFastFetchError(t *testing.T) { for _, err := range []error{ errors.New("nats: disconnected during fetch"), errors.New("nats: connection closed"), errors.New("read tcp: connection reset by peer"), errors.New("write tcp: broken pipe"), errors.New("temporary network unavailable"), errors.New("i/o timeout"), } { if !isTransientFastFetchError(err) { t.Fatalf("isTransientFastFetchError(%q) = false, want true", err.Error()) } } if isTransientFastFetchError(errors.New("permission denied")) { t.Fatal("permission denied should remain an unexpected fetch error") } if isTransientFastFetchError(nil) { t.Fatal("nil should not be treated as transient") } } func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) { t.Setenv("FAST_WRITER_WORKERS", "12") t.Setenv("FAST_WRITER_TDENGINE_MAX_OPEN_CONNS", "") cfg := loadConfig() if cfg.TDengineMaxOpenConns != 1 { t.Fatalf("TDengineMaxOpenConns = %d, want 1", cfg.TDengineMaxOpenConns) } if cfg.TDengineMaxIdleConns != 1 { t.Fatalf("TDengineMaxIdleConns = %d, want 1", cfg.TDengineMaxIdleConns) } } func TestLoadConfigDefaultsTDengineStageDisabled(t *testing.T) { cfg := loadConfig() if cfg.TDengineEnabled { t.Fatal("TDengineEnabled = true, want production default false") } } func TestLoadConfigCanEnableTDengineStageExplicitly(t *testing.T) { for _, value := range []string{"true", "1", "yes", "on"} { t.Run(value, func(t *testing.T) { t.Setenv("FAST_WRITER_TDENGINE_ENABLED", value) cfg := loadConfig() if !cfg.TDengineEnabled { t.Fatalf("TDengineEnabled = false for %q, want explicit true", value) } }) } } func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) { cfg := loadConfig() if got, want := cfg.StreamMaxBytes, int64(20*1024*1024*1024); got != want { t.Fatalf("StreamMaxBytes = %d, want %d", got, want) } if got, want := cfg.StreamEnsureWait, 60*time.Second; got != want { t.Fatalf("StreamEnsureWait = %v, want %v", got, want) } if got, want := cfg.OperationWait, time.Second; got != want { t.Fatalf("OperationWait = %v, want %v", got, want) } if got, want := cfg.FetchWait, 20*time.Millisecond; got != want { t.Fatalf("FetchWait = %v, want %v", got, want) } } func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) { t.Setenv("NATS_STREAM_MAX_BYTES", "1073741824") t.Setenv("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", "90") t.Setenv("FAST_WRITER_OPERATION_TIMEOUT_MS", "1500") t.Setenv("FAST_WRITER_FETCH_WAIT_MS", "45") cfg := loadConfig() if got, want := cfg.StreamMaxBytes, int64(1073741824); got != want { t.Fatalf("StreamMaxBytes = %d, want %d", got, want) } if got, want := cfg.StreamEnsureWait, 90*time.Second; got != want { t.Fatalf("StreamEnsureWait = %v, want %v", got, want) } if got, want := cfg.OperationWait, 1500*time.Millisecond; got != want { t.Fatalf("OperationWait = %v, want %v", got, want) } if got, want := cfg.FetchWait, 45*time.Millisecond; got != want { t.Fatalf("FetchWait = %v, want %v", got, want) } } func TestRecordFastWriterConfigMetrics(t *testing.T) { registry := metrics.NewRegistry() recordFastWriterConfigMetrics(registry, config{ BatchSize: 120, FetchWait: 20 * time.Millisecond, OperationWait: 1500 * time.Millisecond, Workers: 8, }) text := registry.Render() for _, want := range []string{ `vehicle_fast_writer_config{setting="batch_size"} 120`, `vehicle_fast_writer_config{setting="fetch_wait_ms"} 20`, `vehicle_fast_writer_config{setting="operation_timeout_ms"} 1500`, `vehicle_fast_writer_config{setting="workers"} 8`, } { if !strings.Contains(text, want) { t.Fatalf("fast writer config metric missing %s:\n%s", want, text) } } } func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) { t.Setenv("FAST_WRITER_WORKERS", "12") t.Setenv("FAST_WRITER_TDENGINE_MAX_OPEN_CONNS", "4") t.Setenv("FAST_WRITER_TDENGINE_MAX_IDLE_CONNS", "2") cfg := loadConfig() if cfg.TDengineMaxOpenConns != 4 { t.Fatalf("TDengineMaxOpenConns = %d, want 4", cfg.TDengineMaxOpenConns) } if cfg.TDengineMaxIdleConns != 2 { t.Fatalf("TDengineMaxIdleConns = %d, want 2", cfg.TDengineMaxIdleConns) } } type recordingFastAppender struct { count int batchCount int batchRows int err error batchErr error singleErr error failSingleOnCount int onAppendBatch func() } func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error { a.count++ if a.failSingleOnCount > 0 && a.count == a.failSingleOnCount { if a.singleErr != nil { return a.singleErr } return a.err } if a.failSingleOnCount > 0 { return nil } if a.singleErr != nil { return a.singleErr } return a.err } func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelope.FrameEnvelope) error { a.batchCount++ a.batchRows += len(envs) if a.onAppendBatch != nil { a.onAppendBatch() } if a.batchErr != nil { return a.batchErr } return a.err } type recordingFastUpdater struct { count int batchCount int batchRows int err error batchErr error singleErr error failSingleOnCount int } func (u *recordingFastUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { u.count++ if u.failSingleOnCount > 0 && u.count == u.failSingleOnCount { if u.singleErr != nil { return u.singleErr } return u.err } if u.failSingleOnCount > 0 { return nil } if u.singleErr != nil { return u.singleErr } return u.err } func (u *recordingFastUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error { u.batchCount++ u.batchRows += len(envs) if u.batchErr != nil { return u.batchErr } return u.err } type recordingFastSingleUpdater struct { count int err error } func (u *recordingFastSingleUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { u.count++ return u.err } type recordingFastResultUpdater struct { count int resultCount int batchCount int batchResultCount int batchRows int result realtime.FastUpdateResult err error batchErr error singleErr error } func (u *recordingFastResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { u.count++ if u.singleErr != nil { return u.singleErr } return u.err } func (u *recordingFastResultUpdater) FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { u.resultCount++ if u.singleErr != nil { return u.result, u.singleErr } return u.result, u.err } type recordingFastDynamicResultUpdater struct { batchResultCount int batchRows int err error } func (u *recordingFastDynamicResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error { return u.err } func (u *recordingFastDynamicResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { u.batchResultCount++ u.batchRows += len(envs) if u.err != nil { return realtime.FastUpdateResult{}, u.err } return realtime.FastUpdateResult{ EnvelopesSeen: len(envs), EnvelopesUpdated: len(envs), FieldsSeen: len(envs), FieldsWritten: len(envs), }, nil } func (u *recordingFastResultUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error { u.batchCount++ u.batchRows += len(envs) if u.batchErr != nil { return u.batchErr } return u.err } func (u *recordingFastResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) { u.batchResultCount++ u.batchRows += len(envs) if u.batchErr != nil { return u.result, u.batchErr } return u.result, u.err }