feat: build vehicle data platform and production pipeline
This commit is contained in:
lingniu
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
||||
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/realtime"
|
||||
)
|
||||
|
||||
func TestProcessFastMessageWritesTDengineAndRedisBeforeAck(t *testing.T) {
|
||||
@@ -58,8 +59,45 @@ func TestProcessFastMessageDoesNotAckWhenRedisFails(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageUpdatesRedisButDoesNotAckOnTransientTDengineFailure(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-td-transient-single"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{err: errors.New("connection reset by peer")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, appender, updater, msg); err == nil {
|
||||
t.Fatal("processFastMessage() error = nil, want tdengine transient failure")
|
||||
}
|
||||
if updater.count != 1 || ackCount != 0 {
|
||||
t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_last_message_unix_seconds{status="received",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("tdengine transient redis catchup metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("message should not be counted ok because it was not acked:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3"}
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-3", ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli()}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -77,6 +115,14 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_count{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="tdengine",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_stage_unix_seconds{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_bucket{le="+Inf",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_recent_samples{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_last_redis_e2e_unix_seconds{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer stage metric missing %s:\n%s", want, text)
|
||||
@@ -84,6 +130,151 @@ func TestProcessFastMessageRecordsStageDurationMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastWriterRedisE2EDurationSkipsMissingReceiveTime(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
recordFastWriterRedisE2EDuration(registry, "vehicle.raw.go.jt808.v1", envelope.FrameEnvelope{})
|
||||
|
||||
if text := registry.Render(); strings.Contains(text, "vehicle_fast_writer_redis_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_fast_writer_redis_e2e_recent") {
|
||||
t.Fatalf("e2e metric should be skipped when received_at_ms is missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageAcksInvalidJSONAndRecordsMetric(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
updater := &recordingFastUpdater{}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count = %d, want 1", ackCount)
|
||||
}
|
||||
if updater.count != 0 || updater.batchCount != 0 {
|
||||
t.Fatalf("updater counts = single %d batch %d, want 0", updater.count, updater.batchCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("invalid json metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("invalid json should not be counted as ok:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageAcksExplicitNonRawEventKindWithoutWriting(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
EventKind: envelope.EventKindFields,
|
||||
MessageID: "0x0200",
|
||||
VIN: "VIN001",
|
||||
}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{}
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error {
|
||||
ackCount++
|
||||
return nil
|
||||
}}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, appender, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if appender.count != 0 || updater.count != 0 || updater.batchCount != 0 {
|
||||
t.Fatalf("appends=%d updates=%d batch_updates=%d, want no writes", appender.count, updater.count, updater.batchCount)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count=%d, want 1", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_count{stage="ack",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("mismatch metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"}`) {
|
||||
t.Fatalf("mismatched event should not be counted ok:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastMessageRecordsRedisFieldMetrics(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fields"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 4,
|
||||
EnvelopesUpdated: 1,
|
||||
EnvelopesSkippedNonRealtime: 1,
|
||||
EnvelopesSkippedMissingVIN: 1,
|
||||
EnvelopesSkippedMissingVehicleKey: 0,
|
||||
EnvelopesSkippedMissingFields: 1,
|
||||
FieldsSeen: 3,
|
||||
FieldsWritten: 2,
|
||||
FieldsSkippedStale: 1,
|
||||
}}
|
||||
msg := &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }}
|
||||
|
||||
if err := processFastMessage(context.Background(), registry, nil, updater, msg); err != nil {
|
||||
t.Fatalf("processFastMessage() error = %v", err)
|
||||
}
|
||||
if updater.resultCount != 1 {
|
||||
t.Fatalf("FastUpdateWithResult count=%d, want 1", updater.resultCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 4`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_vin",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_non_realtime",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="skipped_missing_fields",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 3`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddFastMetricRecordsLastMessage(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
addFastMetric(registry, "vehicle.raw.go.gb32960.v1", "ok")
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_last_message_unix_seconds{status="ok",subject="vehicle.raw.go.gb32960.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer message metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchAppendsTDengineBatchBeforeRedisAndAck(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-4"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-5"}
|
||||
@@ -149,6 +340,407 @@ func TestProcessFastBatchUsesRedisBatchUpdaterWhenAvailable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallsBackToSinglesAfterNonTransientTDengineBatchFailure(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-td-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{batchErr: errors.New("syntax error in batch insert")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 2 {
|
||||
t.Fatalf("batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 2 || ackCount != 2 {
|
||||
t.Fatalf("updates=%d acks=%d, want 2/2", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="ok"} 1`) {
|
||||
t.Fatalf("tdengine fallback metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchUpdatesRedisButDoesNotAckOnTransientTDengineBatchFailure(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-td-transient"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{batchErr: errors.New("i/o timeout")}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil {
|
||||
t.Fatal("processFastBatch() error = nil, want transient tdengine failure")
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 0 {
|
||||
t.Fatalf("batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 1 || ackCount != 0 {
|
||||
t.Fatalf("updates=%d acks=%d, want 1/0", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="skipped_transient"} 1`,
|
||||
`vehicle_fast_writer_decoupled_updates_total{reason="tdengine_transient",status="ok"} 1`,
|
||||
`vehicle_fast_writer_stage_duration_ms_histogram_bucket{le="+Inf",stage="redis",status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("tdengine transient catchup metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallsBackToRedisSinglesAfterNonTransientRedisBatchFailure(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-redis-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-redis-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{}
|
||||
updater := &recordingFastUpdater{batchErr: errors.New("WRONGTYPE operation against a key holding the wrong kind of value")}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 0 {
|
||||
t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/0", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.count != 2 {
|
||||
t.Fatalf("redis batch updates=%d single updates=%d, want 1/2", updater.batchCount, updater.count)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count=%d, want 2", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="redis",status="ok"} 1`) {
|
||||
t.Fatalf("redis fallback metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchFallbackLeavesFailedAndRemainingMessagesUnacked(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fallback-prefix-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fallback-prefix-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
appender := &recordingFastAppender{
|
||||
batchErr: errors.New("syntax error in batch insert"),
|
||||
singleErr: errors.New("single insert failed"),
|
||||
failSingleOnCount: 2,
|
||||
}
|
||||
updater := &recordingFastSingleUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, appender, updater, msgs); err == nil {
|
||||
t.Fatal("processFastBatch() error = nil, want fallback single write failure")
|
||||
}
|
||||
if appender.batchCount != 1 || appender.count != 2 {
|
||||
t.Fatalf("tdengine batch appends=%d single appends=%d, want 1/2", appender.batchCount, appender.count)
|
||||
}
|
||||
if updater.count != 1 || ackCount != 1 {
|
||||
t.Fatalf("updates=%d acks=%d, want successful prefix 1/1", updater.count, ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if !strings.Contains(text, `vehicle_fast_writer_batch_fallback_total{stage="tdengine",status="error"} 1`) {
|
||||
t.Fatalf("tdengine fallback error metric missing:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisFieldMetrics(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-fields-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-batch-fields-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 2,
|
||||
EnvelopesUpdated: 2,
|
||||
FieldsSeen: 10,
|
||||
FieldsWritten: 7,
|
||||
FieldsSkippedStale: 3,
|
||||
}}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: secondPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchResultCount != 1 || updater.batchCount != 0 {
|
||||
t.Fatalf("batch result count=%d legacy batch count=%d, want 1/0", updater.batchResultCount, updater.batchCount)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count=%d, want 2", ackCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="updated",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 10`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="skipped_stale",subject="vehicle.raw.go.jt808.v1"} 3`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 7`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisE2EByOriginalSubject(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolJT808,
|
||||
VIN: "VIN001",
|
||||
EventID: "evt-batch-e2e-1",
|
||||
ReceivedAtMS: time.Now().Add(-20 * time.Millisecond).UnixMilli(),
|
||||
}
|
||||
second := envelope.FrameEnvelope{
|
||||
Protocol: envelope.ProtocolGB32960,
|
||||
VIN: "VIN002",
|
||||
EventID: "evt-batch-e2e-2",
|
||||
ReceivedAtMS: time.Now().Add(-25 * time.Millisecond).UnixMilli(),
|
||||
}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastResultUpdater{result: realtime.FastUpdateResult{
|
||||
EnvelopesSeen: 2,
|
||||
EnvelopesUpdated: 2,
|
||||
}}
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload},
|
||||
{subject: "vehicle.raw.go.gb32960.v1", data: secondPayload},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.jt808.v1"} `,
|
||||
`vehicle_fast_writer_redis_e2e_recent_p99_ms{subject="vehicle.raw.go.gb32960.v1"} `,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis e2e metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `vehicle_fast_writer_redis_e2e_duration_ms_histogram_count{subject="mixed"}`) {
|
||||
t.Fatalf("redis e2e metric should keep original subjects, not mixed:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchRecordsRedisFieldMetricsByOriginalSubjectForMixedBatch(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-batch-mixed-fields-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN002", EventID: "evt-batch-mixed-fields-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastDynamicResultUpdater{}
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: firstPayload},
|
||||
{subject: "vehicle.raw.go.gb32960.v1", data: secondPayload},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchResultCount != 2 {
|
||||
t.Fatalf("batch result count=%d, want one batch per subject", updater.batchResultCount)
|
||||
}
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_envelopes_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="seen",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_redis_fields_total{status="written",subject="vehicle.raw.go.gb32960.v1"} 1`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("redis field metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
if strings.Contains(text, `subject="mixed"`) {
|
||||
t.Fatalf("mixed subject should not be used for per-subject Redis metrics:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchAcksInvalidJSONWithoutMarkingItOK(t *testing.T) {
|
||||
valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-valid"}
|
||||
validPayload, err := valid.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: []byte(`{bad json`), ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count = %d, want 2", ackCount)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want one valid row", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_messages_total{status="invalid_json",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("batch invalid json metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchSkipsExplicitNonRawEventKindAndWritesValidRows(t *testing.T) {
|
||||
mismatched := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindFields, VIN: "VIN_BAD", EventID: "evt-fields-on-raw"}
|
||||
valid := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, EventKind: envelope.EventKindRaw, VIN: "VIN001", EventID: "evt-valid-raw"}
|
||||
mismatchedPayload, err := mismatched.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
validPayload, err := valid.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: mismatchedPayload, ack: func() error { ackCount++; return nil }},
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: validPayload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if ackCount != 2 {
|
||||
t.Fatalf("ack count = %d, want 2", ackCount)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want only valid row", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_messages_total{status="received",subject="vehicle.raw.go.jt808.v1"} 2`,
|
||||
`vehicle_fast_writer_messages_total{status="event_kind_mismatch",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_messages_total{status="ok",subject="vehicle.raw.go.jt808.v1"} 1`,
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("batch mismatch metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchCanSkipTDengineAppender(t *testing.T) {
|
||||
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-no-td"}
|
||||
payload, err := env.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
updater := &recordingFastUpdater{}
|
||||
ackCount := 0
|
||||
msgs := []*fastMessage{
|
||||
{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { ackCount++; return nil }},
|
||||
}
|
||||
|
||||
if err := processFastBatch(context.Background(), registry, nil, updater, msgs); err != nil {
|
||||
t.Fatalf("processFastBatch() error = %v", err)
|
||||
}
|
||||
if updater.batchCount != 1 || updater.batchRows != 1 {
|
||||
t.Fatalf("redis batch count=%d rows=%d, want 1/1", updater.batchCount, updater.batchRows)
|
||||
}
|
||||
if ackCount != 1 {
|
||||
t.Fatalf("ack count=%d, want 1", ackCount)
|
||||
}
|
||||
text := registry.Render()
|
||||
if strings.Contains(text, `stage="tdengine"`) {
|
||||
t.Fatalf("tdengine stage should not be recorded when appender is nil:\n%s", text)
|
||||
}
|
||||
if !strings.Contains(text, `stage="redis"`) || !strings.Contains(text, `stage="ack"`) {
|
||||
t.Fatalf("redis and ack stages should still be recorded:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-6"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-7"}
|
||||
@@ -208,6 +800,77 @@ func TestProcessFastBatchExposesPendingMetricsDuringAppend(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestProcessFastBatchPendingAggregatesConcurrentWorkers(t *testing.T) {
|
||||
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN001", EventID: "evt-fast-pending-concurrent-1"}
|
||||
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, VIN: "VIN002", EventID: "evt-fast-pending-concurrent-2"}
|
||||
firstPayload, err := first.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secondPayload, err := second.MarshalJSONBytes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := metrics.NewRegistry()
|
||||
firstStarted := make(chan struct{})
|
||||
secondStarted := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
appenderFor := func(started chan struct{}) *recordingFastAppender {
|
||||
return &recordingFastAppender{
|
||||
onAppendBatch: func() {
|
||||
close(started)
|
||||
<-release
|
||||
},
|
||||
}
|
||||
}
|
||||
messages := func(payload []byte, count int) []*fastMessage {
|
||||
out := make([]*fastMessage, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
out = append(out, &fastMessage{subject: "vehicle.raw.go.jt808.v1", data: payload, ack: func() error { return nil }})
|
||||
}
|
||||
return out
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
var firstErr error
|
||||
var secondErr error
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
firstErr = processFastBatch(context.Background(), registry, appenderFor(firstStarted), &recordingFastUpdater{}, messages(firstPayload, 2))
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
secondErr = processFastBatch(context.Background(), registry, appenderFor(secondStarted), &recordingFastUpdater{}, messages(secondPayload, 3))
|
||||
}()
|
||||
<-firstStarted
|
||||
<-secondStarted
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_batch_pending_messages 5`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 5`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("aggregate pending metric missing %s during concurrent append:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if firstErr != nil || secondErr != nil {
|
||||
t.Fatalf("processFastBatch errors = %v / %v", firstErr, secondErr)
|
||||
}
|
||||
text = registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_batch_pending_messages 0`,
|
||||
`vehicle_fast_writer_batch_pending_envelopes 0`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("aggregate pending metric should reset after concurrent append, missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
cfg := config{NATSStream: "VEHICLE_INGEST", NATSDurable: "vehicle-fast-writer"}
|
||||
@@ -230,6 +893,41 @@ func TestRecordFastNATSConsumerInfoMetrics(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsFastWorkerShutdownFetchError(t *testing.T) {
|
||||
if isFastWorkerShutdownFetchError(context.Background(), errors.New("temporary nats failure")) {
|
||||
t.Fatal("temporary failure should not be treated as shutdown")
|
||||
}
|
||||
if !isFastWorkerShutdownFetchError(context.Background(), nats.ErrConnectionClosed) {
|
||||
t.Fatal("connection closed should stop worker without noisy error log")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if !isFastWorkerShutdownFetchError(ctx, errors.New("any fetch error after cancellation")) {
|
||||
t.Fatal("cancelled context should stop worker without noisy error log")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsTransientFastFetchError(t *testing.T) {
|
||||
for _, err := range []error{
|
||||
errors.New("nats: disconnected during fetch"),
|
||||
errors.New("nats: connection closed"),
|
||||
errors.New("read tcp: connection reset by peer"),
|
||||
errors.New("write tcp: broken pipe"),
|
||||
errors.New("temporary network unavailable"),
|
||||
errors.New("i/o timeout"),
|
||||
} {
|
||||
if !isTransientFastFetchError(err) {
|
||||
t.Fatalf("isTransientFastFetchError(%q) = false, want true", err.Error())
|
||||
}
|
||||
}
|
||||
if isTransientFastFetchError(errors.New("permission denied")) {
|
||||
t.Fatal("permission denied should remain an unexpected fetch error")
|
||||
}
|
||||
if isTransientFastFetchError(nil) {
|
||||
t.Fatal("nil should not be treated as transient")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) {
|
||||
t.Setenv("FAST_WRITER_WORKERS", "12")
|
||||
t.Setenv("FAST_WRITER_TDENGINE_MAX_OPEN_CONNS", "")
|
||||
@@ -244,6 +942,28 @@ func TestLoadConfigDefaultsTDenginePoolToSingleConnection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsTDengineStageDisabled(t *testing.T) {
|
||||
cfg := loadConfig()
|
||||
|
||||
if cfg.TDengineEnabled {
|
||||
t.Fatal("TDengineEnabled = true, want production default false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigCanEnableTDengineStageExplicitly(t *testing.T) {
|
||||
for _, value := range []string{"true", "1", "yes", "on"} {
|
||||
t.Run(value, func(t *testing.T) {
|
||||
t.Setenv("FAST_WRITER_TDENGINE_ENABLED", value)
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
if !cfg.TDengineEnabled {
|
||||
t.Fatalf("TDengineEnabled = false for %q, want explicit true", value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) {
|
||||
cfg := loadConfig()
|
||||
|
||||
@@ -256,12 +976,16 @@ func TestLoadConfigDefaultsStreamMaxBytes(t *testing.T) {
|
||||
if got, want := cfg.OperationWait, time.Second; got != want {
|
||||
t.Fatalf("OperationWait = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := cfg.FetchWait, 20*time.Millisecond; got != want {
|
||||
t.Fatalf("FetchWait = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) {
|
||||
t.Setenv("NATS_STREAM_MAX_BYTES", "1073741824")
|
||||
t.Setenv("NATS_STREAM_ENSURE_TIMEOUT_SECONDS", "90")
|
||||
t.Setenv("FAST_WRITER_OPERATION_TIMEOUT_MS", "1500")
|
||||
t.Setenv("FAST_WRITER_FETCH_WAIT_MS", "45")
|
||||
|
||||
cfg := loadConfig()
|
||||
|
||||
@@ -274,6 +998,32 @@ func TestLoadConfigReadsStreamMaxBytesOverride(t *testing.T) {
|
||||
if got, want := cfg.OperationWait, 1500*time.Millisecond; got != want {
|
||||
t.Fatalf("OperationWait = %v, want %v", got, want)
|
||||
}
|
||||
if got, want := cfg.FetchWait, 45*time.Millisecond; got != want {
|
||||
t.Fatalf("FetchWait = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordFastWriterConfigMetrics(t *testing.T) {
|
||||
registry := metrics.NewRegistry()
|
||||
|
||||
recordFastWriterConfigMetrics(registry, config{
|
||||
BatchSize: 120,
|
||||
FetchWait: 20 * time.Millisecond,
|
||||
OperationWait: 1500 * time.Millisecond,
|
||||
Workers: 8,
|
||||
})
|
||||
|
||||
text := registry.Render()
|
||||
for _, want := range []string{
|
||||
`vehicle_fast_writer_config{setting="batch_size"} 120`,
|
||||
`vehicle_fast_writer_config{setting="fetch_wait_ms"} 20`,
|
||||
`vehicle_fast_writer_config{setting="operation_timeout_ms"} 1500`,
|
||||
`vehicle_fast_writer_config{setting="workers"} 8`,
|
||||
} {
|
||||
if !strings.Contains(text, want) {
|
||||
t.Fatalf("fast writer config metric missing %s:\n%s", want, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) {
|
||||
@@ -292,15 +1042,30 @@ func TestLoadConfigReadsTDenginePoolOverride(t *testing.T) {
|
||||
}
|
||||
|
||||
type recordingFastAppender struct {
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
onAppendBatch func()
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
failSingleOnCount int
|
||||
onAppendBatch func()
|
||||
}
|
||||
|
||||
func (a *recordingFastAppender) AppendAll(context.Context, envelope.FrameEnvelope) error {
|
||||
a.count++
|
||||
if a.failSingleOnCount > 0 && a.count == a.failSingleOnCount {
|
||||
if a.singleErr != nil {
|
||||
return a.singleErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
if a.failSingleOnCount > 0 {
|
||||
return nil
|
||||
}
|
||||
if a.singleErr != nil {
|
||||
return a.singleErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
|
||||
@@ -310,24 +1075,45 @@ func (a *recordingFastAppender) AppendAllBatch(_ context.Context, envs []envelop
|
||||
if a.onAppendBatch != nil {
|
||||
a.onAppendBatch()
|
||||
}
|
||||
if a.batchErr != nil {
|
||||
return a.batchErr
|
||||
}
|
||||
return a.err
|
||||
}
|
||||
|
||||
type recordingFastUpdater struct {
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
count int
|
||||
batchCount int
|
||||
batchRows int
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
failSingleOnCount int
|
||||
}
|
||||
|
||||
func (u *recordingFastUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
u.count++
|
||||
if u.failSingleOnCount > 0 && u.count == u.failSingleOnCount {
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
if u.failSingleOnCount > 0 {
|
||||
return nil
|
||||
}
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
||||
u.batchCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.batchErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
@@ -340,3 +1126,73 @@ func (u *recordingFastSingleUpdater) FastUpdate(context.Context, envelope.FrameE
|
||||
u.count++
|
||||
return u.err
|
||||
}
|
||||
|
||||
type recordingFastResultUpdater struct {
|
||||
count int
|
||||
resultCount int
|
||||
batchCount int
|
||||
batchResultCount int
|
||||
batchRows int
|
||||
result realtime.FastUpdateResult
|
||||
err error
|
||||
batchErr error
|
||||
singleErr error
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
u.count++
|
||||
if u.singleErr != nil {
|
||||
return u.singleErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateWithResult(context.Context, envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.resultCount++
|
||||
if u.singleErr != nil {
|
||||
return u.result, u.singleErr
|
||||
}
|
||||
return u.result, u.err
|
||||
}
|
||||
|
||||
type recordingFastDynamicResultUpdater struct {
|
||||
batchResultCount int
|
||||
batchRows int
|
||||
err error
|
||||
}
|
||||
|
||||
func (u *recordingFastDynamicResultUpdater) FastUpdate(context.Context, envelope.FrameEnvelope) error {
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastDynamicResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.batchResultCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.err != nil {
|
||||
return realtime.FastUpdateResult{}, u.err
|
||||
}
|
||||
return realtime.FastUpdateResult{
|
||||
EnvelopesSeen: len(envs),
|
||||
EnvelopesUpdated: len(envs),
|
||||
FieldsSeen: len(envs),
|
||||
FieldsWritten: len(envs),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateBatch(_ context.Context, envs []envelope.FrameEnvelope) error {
|
||||
u.batchCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.batchErr
|
||||
}
|
||||
return u.err
|
||||
}
|
||||
|
||||
func (u *recordingFastResultUpdater) FastUpdateBatchWithResult(_ context.Context, envs []envelope.FrameEnvelope) (realtime.FastUpdateResult, error) {
|
||||
u.batchResultCount++
|
||||
u.batchRows += len(envs)
|
||||
if u.batchErr != nil {
|
||||
return u.result, u.batchErr
|
||||
}
|
||||
return u.result, u.err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user