1244 lines
43 KiB
Go
1244 lines
43 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/segmentio/kafka-go"
|
|
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/metrics"
|
|
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
|
|
)
|
|
|
|
func statTestFieldsEnvelope(env envelope.FrameEnvelope) envelope.FrameEnvelope {
|
|
if env.EventKind == "" {
|
|
env.EventKind = envelope.EventKindFields
|
|
}
|
|
if env.FieldMapping == "" {
|
|
env.FieldMapping = "test-field-mapping.v1"
|
|
}
|
|
if env.ParseStatus == "" {
|
|
env.ParseStatus = envelope.ParseOK
|
|
}
|
|
if len(env.Fields) == 0 {
|
|
switch env.Protocol {
|
|
case envelope.ProtocolGB32960:
|
|
env.Fields = map[string]any{"gb32960.vehicle.total_mileage_km": 4123.9}
|
|
case envelope.ProtocolYutongMQTT:
|
|
env.Fields = map[string]any{"yutong_mqtt.data.total_mileage": 4123900}
|
|
default:
|
|
env.Fields = map[string]any{"jt808.location.total_mileage_km": 4123.9}
|
|
}
|
|
}
|
|
return env
|
|
}
|
|
|
|
func marshalStatTestFieldsEnvelope(t *testing.T, env envelope.FrameEnvelope) []byte {
|
|
t.Helper()
|
|
payload, err := json.Marshal(statTestFieldsEnvelope(env))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func TestProcessStatMessageUsesUncancelledContextForAppendAndCommit(t *testing.T) {
|
|
parent, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
|
|
env := envelope.FrameEnvelope{
|
|
Protocol: envelope.ProtocolJT808,
|
|
MessageID: "0x0200",
|
|
Phone: "13307795425",
|
|
EventTimeMS: 1782918600000,
|
|
ReceivedAtMS: 1782918600000,
|
|
ParseStatus: envelope.ParseOK,
|
|
}
|
|
payload, err := json.Marshal(env)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
|
|
processStatMessage(parent, discardStatLogger{}, nil, appender, committer, kafka.Message{Value: payload})
|
|
|
|
if appender.ctxErr != nil {
|
|
t.Fatalf("appender saw cancelled context: %v", appender.ctxErr)
|
|
}
|
|
if committer.ctxErr != nil {
|
|
t.Fatalf("committer saw cancelled context: %v", committer.ctxErr)
|
|
}
|
|
if appender.count != 1 || committer.count != 1 {
|
|
t.Fatalf("appends=%d commits=%d", appender.count, committer.count)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatMessageRecordsMetrics(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatMessage(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
&contextCheckingStatAppender{result: stats.AppendResult{
|
|
SamplesFound: 1,
|
|
SamplesWritten: 1,
|
|
SamplesSkippedMissingFields: 3,
|
|
SamplesSkippedNonMileageFrame: 4,
|
|
SamplesSkippedSameMileage: 2,
|
|
SamplesAdjustedFutureEventTime: 1,
|
|
SourceTouchesAttempted: 1,
|
|
SourceTouchesWritten: 1,
|
|
SourceTouchesSkippedThrottled: 2,
|
|
SourceTouchesSkippedMissing: 3,
|
|
ProjectionsAttempted: 1,
|
|
ProjectionsWritten: 1,
|
|
ProjectionsSkippedThrottled: 2,
|
|
}, cacheStats: stats.CacheStats{
|
|
LastTotalMileageEntries: 300,
|
|
LastSourceSeenEntries: 3,
|
|
LastProjectionEntries: 280,
|
|
BaselineEntries: 295,
|
|
MaxEntries: 1000000,
|
|
LastCleanupAt: time.Unix(1782918600, 0),
|
|
LastCleanupTotalMileage: 11,
|
|
LastCleanupSourceSeen: 1,
|
|
LastCleanupProjection: 9,
|
|
LastCleanupBaseline: 10,
|
|
TotalMileageEvictions: 4,
|
|
SourceSeenEvictions: 1,
|
|
ProjectionEvictions: 2,
|
|
BaselineEvictions: 3,
|
|
}},
|
|
&contextCheckingStatCommitter{},
|
|
kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 4, Offset: 30, HighWaterMark: 41, Value: payload},
|
|
)
|
|
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="received",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_last_message_unix_seconds{status="received",topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_last_write_unix_seconds{status="ok",topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_write_e2e_recent_samples{topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_last_write_e2e_unix_seconds{topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="found",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_fields",topic="vehicle.fields.go.jt808.v1"} 3`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="skipped_non_mileage_frame",topic="vehicle.fields.go.jt808.v1"} 4`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="skipped_same_mileage",topic="vehicle.fields.go.jt808.v1"} 2`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="event_time_future_adjusted",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_sources_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_sources_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_sources_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 2`,
|
|
`vehicle_stat_sources_total{protocol="JT808",status="skipped_missing_endpoint",topic="vehicle.fields.go.jt808.v1"} 3`,
|
|
`vehicle_stat_projections_total{protocol="JT808",status="attempted",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_projections_total{protocol="JT808",status="written",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_projections_total{protocol="JT808",status="skipped_throttled",topic="vehicle.fields.go.jt808.v1"} 2`,
|
|
`vehicle_stat_cache_entries{cache="last_total_mileage"} 300`,
|
|
`vehicle_stat_cache_entries{cache="source_seen"} 3`,
|
|
`vehicle_stat_cache_entries{cache="projection"} 280`,
|
|
`vehicle_stat_cache_entries{cache="baseline"} 295`,
|
|
`vehicle_stat_cache_max_entries{cache="last_total_mileage"} 1000000`,
|
|
`vehicle_stat_cache_max_entries{cache="source_seen"} 1000000`,
|
|
`vehicle_stat_cache_max_entries{cache="projection"} 1000000`,
|
|
`vehicle_stat_cache_max_entries{cache="baseline"} 1000000`,
|
|
`vehicle_stat_cache_last_cleanup_deleted{cache="last_total_mileage"} 11`,
|
|
`vehicle_stat_cache_last_cleanup_deleted{cache="source_seen"} 1`,
|
|
`vehicle_stat_cache_last_cleanup_deleted{cache="projection"} 9`,
|
|
`vehicle_stat_cache_last_cleanup_deleted{cache="baseline"} 10`,
|
|
`vehicle_stat_cache_evictions_total{cache="last_total_mileage"} 4`,
|
|
`vehicle_stat_cache_evictions_total{cache="source_seen"} 1`,
|
|
`vehicle_stat_cache_evictions_total{cache="projection"} 2`,
|
|
`vehicle_stat_cache_evictions_total{cache="baseline"} 3`,
|
|
`vehicle_stat_cache_last_cleanup_unix_seconds 1782918600`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_last_commit_unix_seconds{status="ok",topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_kafka_lag{partition="4",topic="vehicle.fields.go.jt808.v1"} 10`,
|
|
`vehicle_stat_write_duration_ms_histogram_bucket{le="+Inf",status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_write_duration_ms_histogram_count{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_write_duration_ms_histogram_sum{status="ok",topic="vehicle.fields.go.jt808.v1"}`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("metrics missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRecordStatWriteE2EDurationSkipsMissingReceivedAt(t *testing.T) {
|
|
registry := metrics.NewRegistry()
|
|
|
|
recordStatWriteE2EDuration(registry, kafka.Message{Topic: "vehicle.fields.go.jt808.v1"}, envelope.FrameEnvelope{})
|
|
|
|
text := registry.Render()
|
|
if strings.Contains(text, "vehicle_stat_write_e2e_duration_ms_histogram") || strings.Contains(text, "vehicle_stat_write_e2e_recent") {
|
|
t.Fatalf("unexpected e2e metric without received_at:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchAppendsAllBeforeCommit(t *testing.T) {
|
|
first := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()}
|
|
second := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02", ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli()}
|
|
firstPayload := marshalStatTestFieldsEnvelope(t, first)
|
|
secondPayload := marshalStatTestFieldsEnvelope(t, second)
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, HighWaterMark: 13, Value: firstPayload},
|
|
{Topic: "vehicle.fields.go.gb32960.v1", Partition: 2, Offset: 20, HighWaterMark: 21, Value: secondPayload},
|
|
},
|
|
)
|
|
|
|
if appender.count != 2 {
|
|
t.Fatalf("appends = %d, want 2", appender.count)
|
|
}
|
|
if committer.count != 1 {
|
|
t.Fatalf("commit calls = %d, want 1", committer.count)
|
|
}
|
|
if committer.messageCount != 2 {
|
|
t.Fatalf("committed messages = %d, want 2", committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_batch_pending_messages 0`,
|
|
`vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
`vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_write_e2e_duration_ms_histogram_count{topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
`vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.jt808.v1"} `,
|
|
`vehicle_stat_write_e2e_recent_p99_ms{topic="vehicle.fields.go.gb32960.v1"} `,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("batch metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchKeepsPendingGaugePerWorker(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatchForWorker(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
&contextCheckingStatAppender{},
|
|
&contextCheckingStatCommitter{},
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}},
|
|
metrics.Labels{"worker": "2"},
|
|
)
|
|
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_batch_pending_messages{worker="2"} 0`) {
|
|
t.Fatalf("worker pending gauge missing or not reset:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchSamplesCacheMetricsOncePerBatch(t *testing.T) {
|
|
payload := marshalStatTestFieldsEnvelope(t, envelope.FrameEnvelope{
|
|
Protocol: envelope.ProtocolJT808,
|
|
VIN: "LNBVIN00000000001",
|
|
EventTimeMS: 1782918600000,
|
|
ReceivedAtMS: time.Now().Add(-time.Second).UnixMilli(),
|
|
Fields: map[string]any{
|
|
"jt808.location.total_mileage_km": 4123.9,
|
|
},
|
|
})
|
|
appender := &contextCheckingStatAppender{
|
|
result: stats.AppendResult{SamplesFound: 1, SamplesWritten: 1},
|
|
cacheStats: stats.CacheStats{
|
|
LastTotalMileageEntries: 12,
|
|
MaxEntries: 99,
|
|
},
|
|
}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
|
|
},
|
|
)
|
|
|
|
if appender.cacheStatsCalls != 1 {
|
|
t.Fatalf("CacheStats calls = %d, want 1 per batch", appender.cacheStatsCalls)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_cache_entries{cache="last_total_mileage"} 12`) {
|
|
t.Fatalf("cache metric missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchCommitsInvalidJSONWithoutWriteMetric(t *testing.T) {
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Value: []byte("{bad json")}},
|
|
)
|
|
|
|
if appender.count != 0 {
|
|
t.Fatalf("appends = %d, want 0", appender.count)
|
|
}
|
|
if committer.count != 1 {
|
|
t.Fatalf("commit calls = %d, want 1", committer.count)
|
|
}
|
|
if committer.messageCount != 1 {
|
|
t.Fatalf("committed messages = %d, want 1", committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("invalid json metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
if strings.Contains(text, `vehicle_stat_writes_total`) {
|
|
t.Fatalf("invalid json should not record write metric:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchSkipsKnownFieldsTopicProtocolMismatch(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload, err := json.Marshal(env)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.gb32960.v1", Partition: 1, Offset: 10, Value: payload}},
|
|
)
|
|
|
|
if appender.count != 0 {
|
|
t.Fatalf("appends = %d, want 0", appender.count)
|
|
}
|
|
if committer.count != 1 || committer.messageCount != 1 {
|
|
t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="protocol_topic_mismatch",topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("protocol mismatch metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
if strings.Contains(text, `vehicle_stat_writes_total`) {
|
|
t.Fatalf("protocol mismatch should not record write metric:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchSkipsExplicitNonFieldsEventKind(t *testing.T) {
|
|
env := envelope.FrameEnvelope{
|
|
EventKind: envelope.EventKindRaw,
|
|
Protocol: envelope.ProtocolJT808,
|
|
Phone: "13307795425",
|
|
MessageID: "0x0200",
|
|
}
|
|
payload, err := json.Marshal(env)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}},
|
|
)
|
|
|
|
if appender.count != 0 {
|
|
t.Fatalf("appends = %d, want 0", appender.count)
|
|
}
|
|
if committer.count != 1 || committer.messageCount != 1 {
|
|
t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="event_kind_mismatch",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("event kind mismatch metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
if strings.Contains(text, `vehicle_stat_writes_total`) {
|
|
t.Fatalf("event kind mismatch should not record write metric:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchSkipsFieldsContractViolation(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
env envelope.FrameEnvelope
|
|
status string
|
|
}{
|
|
{
|
|
name: "missing_field_mapping",
|
|
env: envelope.FrameEnvelope{
|
|
EventKind: envelope.EventKindFields,
|
|
Protocol: envelope.ProtocolJT808,
|
|
Fields: map[string]any{"jt808.location.total_mileage_km": 4123.9},
|
|
},
|
|
status: "missing_field_mapping",
|
|
},
|
|
{
|
|
name: "missing_fields",
|
|
env: envelope.FrameEnvelope{
|
|
EventKind: envelope.EventKindFields,
|
|
Protocol: envelope.ProtocolJT808,
|
|
FieldMapping: "test-field-mapping.v1",
|
|
},
|
|
status: "missing_fields",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
payload, err := json.Marshal(tt.env)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}},
|
|
)
|
|
|
|
if appender.count != 0 {
|
|
t.Fatalf("appends = %d, want 0", appender.count)
|
|
}
|
|
if committer.count != 1 || committer.messageCount != 1 {
|
|
t.Fatalf("commits=%d messages=%d, want 1/1", committer.count, committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="` + tt.status + `",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("contract violation metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
if strings.Contains(text, `vehicle_stat_writes_total`) {
|
|
t.Fatalf("contract violation should not record write metric:\n%s", text)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchExposesPendingMessagesDuringAppend(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
registry := metrics.NewRegistry()
|
|
appender := &contextCheckingStatAppender{
|
|
onAppend: func() {
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_batch_pending_messages 2`) {
|
|
t.Fatalf("pending batch metric missing while appending:\n%s", text)
|
|
}
|
|
},
|
|
}
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
&contextCheckingStatCommitter{},
|
|
[]kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Value: payload},
|
|
},
|
|
)
|
|
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_batch_pending_messages 0`) {
|
|
t.Fatalf("pending batch metric should reset after append:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchDoesNotCommitWhenAppendFails(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
appender := &contextCheckingStatAppender{err: errTestStatAppend}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Value: payload}},
|
|
)
|
|
|
|
if committer.count != 0 {
|
|
t.Fatalf("commit calls = %d, want 0", committer.count)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_writes_total{status="error",topic="vehicle.fields.go.jt808.v1"} 1`) {
|
|
t.Fatalf("write error metric missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchCommitsMissingTimeSample(t *testing.T) {
|
|
env := envelope.FrameEnvelope{
|
|
Protocol: envelope.ProtocolJT808,
|
|
VIN: "LA9GG64L7PBAF4001",
|
|
Phone: "13307765812",
|
|
SourceEndpoint: "115.231.168.135:20215",
|
|
Fields: map[string]any{
|
|
"jt808.location.total_mileage_km": 4123.9,
|
|
},
|
|
}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
resultAppender := &contextCheckingStatAppender{
|
|
result: stats.AppendResult{
|
|
SamplesSkippedMissingTime: 1,
|
|
SourceTouchesAttempted: 1,
|
|
SourceTouchesWritten: 1,
|
|
},
|
|
}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
resultAppender,
|
|
committer,
|
|
[]kafka.Message{{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload}},
|
|
)
|
|
|
|
if committer.messageCount != 1 {
|
|
t.Fatalf("committed messages = %d, want 1", committer.messageCount)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_samples_total{protocol="JT808",status="skipped_missing_time",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("missing time metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchCommitsProcessedPrefixWhenLaterAppendFails(t *testing.T) {
|
|
success := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
failed := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
|
|
successPayload := marshalStatTestFieldsEnvelope(t, success)
|
|
failedPayload := marshalStatTestFieldsEnvelope(t, failed)
|
|
appender := &contextCheckingStatAppender{failOnCount: 2, err: errTestStatAppend}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
|
|
outcome := processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: []byte("{bad json")},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: successPayload},
|
|
{Topic: "vehicle.fields.go.gb32960.v1", Partition: 2, Offset: 20, Value: failedPayload},
|
|
},
|
|
)
|
|
|
|
if appender.count != 2 {
|
|
t.Fatalf("appends = %d, want 2 valid appends", appender.count)
|
|
}
|
|
if committer.messageCount != 2 {
|
|
t.Fatalf("committed messages = %d, want processed prefix only", committer.messageCount)
|
|
}
|
|
if got := committedOffsets(outcome.retryMessages); len(got) != 1 || got[0] != 20 {
|
|
t.Fatalf("retry offsets = %#v, want [20]", got)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_kafka_messages_total{status="invalid_json",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_writes_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`,
|
|
`vehicle_stat_writes_total{status="error",topic="vehicle.fields.go.gb32960.v1"} 1`,
|
|
`vehicle_stat_kafka_commits_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 2`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("processed prefix metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchReliablyRetriesFailedSuffix(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
appender := &contextCheckingStatAppender{failOnCount: 2, err: errTestStatAppend}
|
|
committer := &contextCheckingStatCommitter{}
|
|
registry := metrics.NewRegistry()
|
|
messages := []kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12, Value: payload},
|
|
}
|
|
|
|
processStatBatchReliably(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
messages,
|
|
time.Nanosecond,
|
|
)
|
|
|
|
if appender.count != 4 {
|
|
t.Fatalf("append attempts = %d, want success + failed + retried suffix", appender.count)
|
|
}
|
|
if got := committedOffsets(committer.messages); len(got) != 3 || got[0] != 10 || got[1] != 11 || got[2] != 12 {
|
|
t.Fatalf("committed offsets = %#v, want [10 11 12]", got)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_batch_retries_total{reason="write_error"} 1`) {
|
|
t.Fatalf("write retry metric missing:\n%s", text)
|
|
}
|
|
if !strings.Contains(text, `vehicle_stat_retry_pending_messages 0`) {
|
|
t.Fatalf("retry pending gauge should reset:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchReliablyQuarantinesPermanentFailureAndContinues(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
appender := &contextCheckingStatAppender{failOnCount: 1, err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
|
|
committer := &contextCheckingStatCommitter{}
|
|
quarantiner := &recordingStatQuarantiner{}
|
|
registry := metrics.NewRegistry()
|
|
messages := []kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
|
|
}
|
|
|
|
processStatBatchReliablyForWorker(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
quarantiner,
|
|
messages,
|
|
time.Nanosecond,
|
|
nil,
|
|
)
|
|
|
|
if appender.count != 2 {
|
|
t.Fatalf("append attempts = %d, want failed message once and following message once", appender.count)
|
|
}
|
|
if quarantiner.count != 1 || quarantiner.message.Offset != 10 {
|
|
t.Fatalf("quarantine = %d/%d, want one message at offset 10", quarantiner.count, quarantiner.message.Offset)
|
|
}
|
|
if got := committedOffsets(committer.messages); len(got) != 2 || got[0] != 10 || got[1] != 11 {
|
|
t.Fatalf("committed offsets = %#v, want [10 11]", got)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_quarantine_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`) {
|
|
t.Fatalf("quarantine success metric missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestFileStatMessageQuarantinerPersistsOriginalKafkaMessage(t *testing.T) {
|
|
dir := t.TempDir()
|
|
message := kafka.Message{
|
|
Topic: "vehicle.fields.go.jt808.v1",
|
|
Partition: 2,
|
|
Offset: 5094052,
|
|
Key: []byte("vehicle-key"),
|
|
Value: []byte(`{"event_id":"poison:fields"}`),
|
|
}
|
|
wantErr := errors.New("permanent mysql error")
|
|
|
|
if err := (fileStatMessageQuarantiner{dir: dir}).Quarantine(context.Background(), message, wantErr); err != nil {
|
|
t.Fatalf("Quarantine() error = %v", err)
|
|
}
|
|
payload, err := os.ReadFile(filepath.Join(dir, "vehicle.fields.go.jt808.v1-2-5094052.json"))
|
|
if err != nil {
|
|
t.Fatalf("read quarantine record: %v", err)
|
|
}
|
|
var record quarantinedStatMessage
|
|
if err := json.Unmarshal(payload, &record); err != nil {
|
|
t.Fatalf("unmarshal quarantine record: %v", err)
|
|
}
|
|
if record.Topic != message.Topic || record.Partition != message.Partition || record.Offset != message.Offset || string(record.Key) != string(message.Key) || string(record.Value) != string(message.Value) || record.Error != wantErr.Error() {
|
|
t.Fatalf("quarantine record = %#v, want original message and error", record)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchReliablyRetriesCommitWithoutReappending(t *testing.T) {
|
|
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
payload := marshalStatTestFieldsEnvelope(t, env)
|
|
appender := &contextCheckingStatAppender{}
|
|
committer := &contextCheckingStatCommitter{err: errors.New("commit failed"), failOnCount: 1}
|
|
registry := metrics.NewRegistry()
|
|
messages := []kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
|
|
}
|
|
|
|
processStatBatchReliably(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
registry,
|
|
appender,
|
|
committer,
|
|
messages,
|
|
time.Nanosecond,
|
|
)
|
|
|
|
if appender.count != 2 {
|
|
t.Fatalf("append attempts = %d, want 2 without replay after commit failure", appender.count)
|
|
}
|
|
if committer.count != 2 {
|
|
t.Fatalf("commit attempts = %d, want initial failure and one retry", committer.count)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_batch_retries_total{reason="commit_error"} 1`) {
|
|
t.Fatalf("commit retry metric missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestProcessStatBatchDoesNotCommitAcrossOffsetGapAfterFailure(t *testing.T) {
|
|
success := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
|
|
failed := envelope.FrameEnvelope{Protocol: envelope.ProtocolGB32960, VIN: "VIN001", MessageID: "0x02"}
|
|
successPayload := marshalStatTestFieldsEnvelope(t, success)
|
|
failedPayload := marshalStatTestFieldsEnvelope(t, failed)
|
|
appender := &contextCheckingStatAppender{failOnCount: 1, err: errTestStatAppend}
|
|
committer := &contextCheckingStatCommitter{}
|
|
|
|
processStatBatch(
|
|
context.Background(),
|
|
discardStatLogger{},
|
|
metrics.NewRegistry(),
|
|
appender,
|
|
committer,
|
|
[]kafka.Message{
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: successPayload},
|
|
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12, Value: failedPayload},
|
|
},
|
|
)
|
|
|
|
if committer.count != 0 {
|
|
t.Fatalf("commit calls = %d, want 0 because failed first processed offset must not be skipped", committer.count)
|
|
}
|
|
}
|
|
|
|
func TestProcessedPrefixMessagesByPartitionStopsAtGap(t *testing.T) {
|
|
items := []statBatchItem{
|
|
{message: kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10}, processed: true},
|
|
{message: kafka.Message{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 12}, processed: true},
|
|
}
|
|
|
|
got := processedPrefixMessagesByPartition(items)
|
|
if len(got) != 1 || got[0].Offset != 10 {
|
|
t.Fatalf("processed prefix = %#v, want only offset 10", got)
|
|
}
|
|
}
|
|
|
|
func TestRecordStatSampleMetricsLabelsUnknownProtocol(t *testing.T) {
|
|
registry := metrics.NewRegistry()
|
|
|
|
recordStatSampleMetrics(registry, kafka.Message{Topic: "vehicle.fields.go.unknown.v1"}, "", stats.AppendResult{
|
|
SamplesSkippedMissingVIN: 1,
|
|
})
|
|
recordStatProjectionMetrics(registry, kafka.Message{Topic: "vehicle.fields.go.unknown.v1"}, "", stats.AppendResult{
|
|
ProjectionsSkippedThrottled: 1,
|
|
})
|
|
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_samples_total{protocol="UNKNOWN",status="skipped_missing_vin",topic="vehicle.fields.go.unknown.v1"} 1`,
|
|
`vehicle_stat_projections_total{protocol="UNKNOWN",status="skipped_throttled",topic="vehicle.fields.go.unknown.v1"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("metrics missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderRetriesTransientMySQLDeadlock(t *testing.T) {
|
|
registry := metrics.NewRegistry()
|
|
delegate := &contextCheckingStatAppender{
|
|
failOnCount: 1,
|
|
err: errors.New("Error 1213: Deadlock found when trying to get lock"),
|
|
result: stats.AppendResult{SamplesWritten: 1},
|
|
}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry}
|
|
|
|
result, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
|
|
|
if err != nil {
|
|
t.Fatalf("AppendWithResult() error = %v", err)
|
|
}
|
|
if delegate.count != 2 {
|
|
t.Fatalf("append attempts = %d, want 2", delegate.count)
|
|
}
|
|
if result.SamplesWritten != 1 {
|
|
t.Fatalf("result = %#v, want successful result", result)
|
|
}
|
|
text := registry.Render()
|
|
if !strings.Contains(text, `vehicle_stat_write_retries_total{operation="single",status="retry"} 1`) {
|
|
t.Fatalf("retry metric missing:\n%s", text)
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderRetriesTransientMySQLConnectionFailure(t *testing.T) {
|
|
delegate := &contextCheckingStatAppender{
|
|
failOnCount: 1,
|
|
err: errors.New("driver: bad connection: read tcp 10.0.0.1:3306: connection reset by peer"),
|
|
result: stats.AppendResult{SamplesWritten: 1},
|
|
}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 3}
|
|
|
|
result, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
|
|
|
if err != nil {
|
|
t.Fatalf("AppendWithResult() error = %v", err)
|
|
}
|
|
if delegate.count != 2 {
|
|
t.Fatalf("append attempts = %d, want 2", delegate.count)
|
|
}
|
|
if result.SamplesWritten != 1 {
|
|
t.Fatalf("result = %#v, want successful retry result", result)
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderRecordsExhaustedTransientError(t *testing.T) {
|
|
registry := metrics.NewRegistry()
|
|
delegate := &contextCheckingStatAppender{err: errors.New("Error 1205: Lock wait timeout exceeded")}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 2, registry: registry}
|
|
|
|
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
|
|
|
if err == nil {
|
|
t.Fatal("AppendWithResult() error = nil, want exhausted transient error")
|
|
}
|
|
if delegate.count != 2 {
|
|
t.Fatalf("append attempts = %d, want 2", delegate.count)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_write_retries_total{operation="single",status="retry"} 1`,
|
|
`vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("retry metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderRetriesQuarantinableDataErrorBeforeIsolation(t *testing.T) {
|
|
registry := metrics.NewRegistry()
|
|
delegate := &contextCheckingStatAppender{err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry}
|
|
|
|
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
|
|
|
if err == nil {
|
|
t.Fatal("AppendWithResult() error = nil, want exhausted data error")
|
|
}
|
|
if delegate.count != 3 {
|
|
t.Fatalf("append attempts = %d, want configured finite retries", delegate.count)
|
|
}
|
|
text := registry.Render()
|
|
for _, want := range []string{
|
|
`vehicle_stat_write_retries_total{operation="single",status="retry"} 2`,
|
|
`vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
|
|
} {
|
|
if !strings.Contains(text, want) {
|
|
t.Fatalf("retry metric missing %s:\n%s", want, text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
|
|
wantErr := errors.New("validation failed")
|
|
delegate := &contextCheckingStatAppender{err: wantErr}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 3}
|
|
|
|
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
|
|
|
|
if !errors.Is(err, wantErr) {
|
|
t.Fatalf("AppendWithResult() error = %v, want %v", err, wantErr)
|
|
}
|
|
if delegate.count != 1 {
|
|
t.Fatalf("append attempts = %d, want 1", delegate.count)
|
|
}
|
|
}
|
|
|
|
func TestIsQuarantinableStatErrorOnlyAcceptsDeterministicRowDataFailures(t *testing.T) {
|
|
for _, err := range []error{
|
|
errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null"),
|
|
errors.New("Error 1264: Out of range value for column 'daily_mileage_km'"),
|
|
errors.New("Error 1265: Data truncated for column 'daily_mileage_km'"),
|
|
errors.New("Error 1292: Incorrect datetime value"),
|
|
errors.New("Error 1366: Incorrect decimal value"),
|
|
errors.New("Error 1406: Data too long for column"),
|
|
errors.New("Error 3819: Check constraint is violated"),
|
|
} {
|
|
if !isQuarantinableStatError(err) {
|
|
t.Fatalf("isQuarantinableStatError(%q) = false, want true", err.Error())
|
|
}
|
|
}
|
|
for _, err := range []error{
|
|
errors.New("Error 1054: Unknown column 'daily_mileage_km'"),
|
|
errors.New("Error 1142: INSERT command denied"),
|
|
errors.New("validation failed"),
|
|
context.Canceled,
|
|
context.DeadlineExceeded,
|
|
nil,
|
|
} {
|
|
if isQuarantinableStatError(err) {
|
|
t.Fatalf("isQuarantinableStatError(%v) = true, want false", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestIsTransientMySQLStatError(t *testing.T) {
|
|
for _, err := range []error{
|
|
errors.New("dial tcp 127.0.0.1:3306: connection refused"),
|
|
errors.New("read tcp: connection reset by peer"),
|
|
errors.New("write tcp: broken pipe"),
|
|
errors.New("driver: bad connection"),
|
|
errors.New("invalid connection"),
|
|
errors.New("i/o timeout"),
|
|
errors.New("EOF"),
|
|
errors.New("server is down"),
|
|
errors.New("network is unreachable"),
|
|
} {
|
|
if !isTransientMySQLStatError(err) {
|
|
t.Fatalf("isTransientMySQLStatError(%q) = false, want true", err.Error())
|
|
}
|
|
}
|
|
for _, err := range []error{
|
|
context.Canceled,
|
|
context.DeadlineExceeded,
|
|
errors.New("duplicate key conflict"),
|
|
nil,
|
|
} {
|
|
if isTransientMySQLStatError(err) {
|
|
t.Fatalf("isTransientMySQLStatError(%v) = true, want false", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRetryStatAppenderForwardsCacheStats(t *testing.T) {
|
|
delegate := &contextCheckingStatAppender{cacheStats: stats.CacheStats{BaselineEntries: 12, MaxEntries: 99}}
|
|
appender := retryStatAppender{delegate: delegate, attempts: 3}
|
|
|
|
got := appender.CacheStats()
|
|
|
|
if got.BaselineEntries != 12 || got.MaxEntries != 99 {
|
|
t.Fatalf("CacheStats() = %+v, want delegated stats", got)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigDefaultsToGoFieldsTopics(t *testing.T) {
|
|
cfg := loadConfig()
|
|
|
|
got := strings.Join(cfg.KafkaTopics, ",")
|
|
want := "vehicle.fields.go.gb32960.v1,vehicle.fields.go.jt808.v1,vehicle.fields.go.yutong-mqtt.v1"
|
|
if got != want {
|
|
t.Fatalf("KafkaTopics = %q, want %q", got, want)
|
|
}
|
|
if cfg.BatchSize != 200 {
|
|
t.Fatalf("BatchSize = %d, want 200", cfg.BatchSize)
|
|
}
|
|
if cfg.BatchWait != 20 {
|
|
t.Fatalf("BatchWait = %d, want 20", cfg.BatchWait)
|
|
}
|
|
if cfg.CacheMaxEntries != 1000000 {
|
|
t.Fatalf("CacheMaxEntries = %d, want 1000000", cfg.CacheMaxEntries)
|
|
}
|
|
if cfg.Workers != 3 {
|
|
t.Fatalf("Workers = %d, want 3", cfg.Workers)
|
|
}
|
|
if cfg.BaselineMissTTL != time.Minute {
|
|
t.Fatalf("BaselineMissTTL = %s, want 1m", cfg.BaselineMissTTL)
|
|
}
|
|
if cfg.BaselineHitTTL != 5*time.Minute {
|
|
t.Fatalf("BaselineHitTTL = %s, want 5m", cfg.BaselineHitTTL)
|
|
}
|
|
if cfg.RetryAttempts != 3 {
|
|
t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts)
|
|
}
|
|
if cfg.RetryDelay != 20*time.Millisecond {
|
|
t.Fatalf("RetryDelay = %s, want 20ms", cfg.RetryDelay)
|
|
}
|
|
if !cfg.NormalizePlatformSourcesOnStart {
|
|
t.Fatal("NormalizePlatformSourcesOnStart = false, want default true")
|
|
}
|
|
if cfg.NormalizePlatformSourcesTimeout != 30*time.Second {
|
|
t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 30s", cfg.NormalizePlatformSourcesTimeout)
|
|
}
|
|
}
|
|
|
|
func TestConfigValidateRejectsRawTopics(t *testing.T) {
|
|
cfg := config{KafkaTopics: []string{"vehicle.raw.go.jt808.v1"}}
|
|
|
|
err := cfg.Validate()
|
|
|
|
if err == nil {
|
|
t.Fatal("Validate() error = nil, want raw topic rejection")
|
|
}
|
|
if !strings.Contains(err.Error(), "fields topics only") {
|
|
t.Fatalf("Validate() error = %q, want fields topic hint", err)
|
|
}
|
|
}
|
|
|
|
func TestConfigValidateRequiresTopics(t *testing.T) {
|
|
cfg := config{}
|
|
|
|
err := cfg.Validate()
|
|
|
|
if err == nil {
|
|
t.Fatal("Validate() error = nil, want missing topic rejection")
|
|
}
|
|
if !strings.Contains(err.Error(), "KAFKA_TOPICS") {
|
|
t.Fatalf("Validate() error = %q, want KAFKA_TOPICS hint", err)
|
|
}
|
|
}
|
|
|
|
func TestConfigValidateRequiresPositiveWorkers(t *testing.T) {
|
|
cfg := config{
|
|
KafkaTopics: []string{"vehicle.fields.go.jt808.v1"},
|
|
Workers: 0,
|
|
}
|
|
|
|
err := cfg.Validate()
|
|
|
|
if err == nil {
|
|
t.Fatal("Validate() error = nil, want worker count rejection")
|
|
}
|
|
if !strings.Contains(err.Error(), "STATS_WORKERS") {
|
|
t.Fatalf("Validate() error = %q, want STATS_WORKERS hint", err)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
|
|
t.Setenv("STATS_PROJECT_INTERVAL_SECONDS", "7")
|
|
t.Setenv("STATS_SOURCE_TOUCH_INTERVAL_SECONDS", "11")
|
|
t.Setenv("STATS_CACHE_RETENTION_HOURS", "48")
|
|
t.Setenv("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", "300")
|
|
t.Setenv("STATS_BASELINE_MISS_TTL_SECONDS", "45")
|
|
t.Setenv("STATS_BASELINE_HIT_TTL_SECONDS", "180")
|
|
t.Setenv("STATS_CACHE_MAX_ENTRIES", "12345")
|
|
|
|
cfg := loadConfig()
|
|
|
|
if cfg.ProjectInterval != 7*time.Second {
|
|
t.Fatalf("ProjectInterval = %s, want 7s", cfg.ProjectInterval)
|
|
}
|
|
if cfg.SourceTouchInterval != 11*time.Second {
|
|
t.Fatalf("SourceTouchInterval = %s, want 11s", cfg.SourceTouchInterval)
|
|
}
|
|
if cfg.CacheRetention != 48*time.Hour {
|
|
t.Fatalf("CacheRetention = %s, want 48h", cfg.CacheRetention)
|
|
}
|
|
if cfg.CacheCleanupInterval != 5*time.Minute {
|
|
t.Fatalf("CacheCleanupInterval = %s, want 5m", cfg.CacheCleanupInterval)
|
|
}
|
|
if cfg.BaselineMissTTL != 45*time.Second {
|
|
t.Fatalf("BaselineMissTTL = %s, want 45s", cfg.BaselineMissTTL)
|
|
}
|
|
if cfg.BaselineHitTTL != 3*time.Minute {
|
|
t.Fatalf("BaselineHitTTL = %s, want 3m", cfg.BaselineHitTTL)
|
|
}
|
|
if cfg.CacheMaxEntries != 12345 {
|
|
t.Fatalf("CacheMaxEntries = %d, want 12345", cfg.CacheMaxEntries)
|
|
}
|
|
}
|
|
|
|
func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
|
|
t.Setenv("STATS_WORKERS", "5")
|
|
t.Setenv("STATS_BATCH_SIZE", "500")
|
|
t.Setenv("STATS_BATCH_WAIT_MS", "250")
|
|
t.Setenv("STATS_RETRY_ATTEMPTS", "5")
|
|
t.Setenv("STATS_RETRY_DELAY_MS", "33")
|
|
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "false")
|
|
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", "17")
|
|
t.Setenv("STATS_QUARANTINE_DIR", "/tmp/stat-writer-quarantine-test")
|
|
|
|
cfg := loadConfig()
|
|
|
|
if cfg.Workers != 5 {
|
|
t.Fatalf("Workers = %d, want 5", cfg.Workers)
|
|
}
|
|
if cfg.BatchSize != 500 {
|
|
t.Fatalf("BatchSize = %d, want 500", cfg.BatchSize)
|
|
}
|
|
if cfg.BatchWait != 250 {
|
|
t.Fatalf("BatchWait = %d, want 250", cfg.BatchWait)
|
|
}
|
|
if cfg.RetryAttempts != 5 {
|
|
t.Fatalf("RetryAttempts = %d, want 5", cfg.RetryAttempts)
|
|
}
|
|
if cfg.RetryDelay != 33*time.Millisecond {
|
|
t.Fatalf("RetryDelay = %s, want 33ms", cfg.RetryDelay)
|
|
}
|
|
if cfg.NormalizePlatformSourcesOnStart {
|
|
t.Fatal("NormalizePlatformSourcesOnStart = true, want env override false")
|
|
}
|
|
if cfg.NormalizePlatformSourcesTimeout != 17*time.Second {
|
|
t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 17s", cfg.NormalizePlatformSourcesTimeout)
|
|
}
|
|
if cfg.QuarantineDir != "/tmp/stat-writer-quarantine-test" {
|
|
t.Fatalf("QuarantineDir = %q, want env override", cfg.QuarantineDir)
|
|
}
|
|
}
|
|
|
|
type contextCheckingStatAppender struct {
|
|
ctxErr error
|
|
count int
|
|
result stats.AppendResult
|
|
cacheStats stats.CacheStats
|
|
cacheStatsCalls int
|
|
err error
|
|
failOnCount int
|
|
onAppend func()
|
|
}
|
|
|
|
func (a *contextCheckingStatAppender) Append(ctx context.Context, _ envelope.FrameEnvelope) error {
|
|
a.ctxErr = ctx.Err()
|
|
a.count++
|
|
if a.onAppend != nil {
|
|
a.onAppend()
|
|
}
|
|
if a.ctxErr != nil {
|
|
return a.ctxErr
|
|
}
|
|
if a.failOnCount > 0 {
|
|
if a.count == a.failOnCount {
|
|
return a.err
|
|
}
|
|
return nil
|
|
}
|
|
return a.err
|
|
}
|
|
|
|
func (a *contextCheckingStatAppender) AppendWithResult(ctx context.Context, env envelope.FrameEnvelope) (stats.AppendResult, error) {
|
|
return a.result, a.Append(ctx, env)
|
|
}
|
|
|
|
func (a *contextCheckingStatAppender) CacheStats() stats.CacheStats {
|
|
a.cacheStatsCalls++
|
|
return a.cacheStats
|
|
}
|
|
|
|
type contextCheckingStatCommitter struct {
|
|
ctxErr error
|
|
count int
|
|
messageCount int
|
|
messages []kafka.Message
|
|
err error
|
|
failOnCount int
|
|
}
|
|
|
|
type recordingStatQuarantiner struct {
|
|
count int
|
|
message kafka.Message
|
|
err error
|
|
}
|
|
|
|
func (q *recordingStatQuarantiner) Quarantine(_ context.Context, message kafka.Message, _ error) error {
|
|
q.count++
|
|
q.message = message
|
|
return q.err
|
|
}
|
|
|
|
func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
|
|
c.ctxErr = ctx.Err()
|
|
c.count++
|
|
c.messageCount += len(messages)
|
|
c.messages = append(c.messages, messages...)
|
|
if c.ctxErr != nil {
|
|
return c.ctxErr
|
|
}
|
|
if c.err != nil && (c.failOnCount == 0 || c.count == c.failOnCount) {
|
|
return c.err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type discardStatLogger struct{}
|
|
|
|
func (discardStatLogger) Error(string, ...any) {}
|
|
func (discardStatLogger) Warn(string, ...any) {}
|
|
|
|
var errTestStatAppend = errors.New("test stat append failed")
|
|
|
|
func committedOffsets(messages []kafka.Message) []int64 {
|
|
offsets := make([]int64, len(messages))
|
|
for index, message := range messages {
|
|
offsets[index] = message.Offset
|
|
}
|
|
return offsets
|
|
}
|