shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): harden telemetry pipeline and unify Semi UI workspaces

Browse Source
This commit is contained in:
lingniu
2026-07-18 00:26:36 +08:00
parent 65b4e4f055
commit 159c80b0ae
136 changed files with 21616 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
go/vehicle-gateway/cmd/stat-writer/main_test.go
View File

@@ -4,6 +4,8 @@ import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -664,6 +666,72 @@ func TestProcessStatBatchReliablyRetriesFailedSuffix(t *testing.T) {
}
}
func TestProcessStatBatchReliablyQuarantinesPermanentFailureAndContinues(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
payload := marshalStatTestFieldsEnvelope(t, env)
appender := &contextCheckingStatAppender{failOnCount: 1, err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
committer := &contextCheckingStatCommitter{}
quarantiner := &recordingStatQuarantiner{}
registry := metrics.NewRegistry()
messages := []kafka.Message{
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 10, Value: payload},
{Topic: "vehicle.fields.go.jt808.v1", Partition: 1, Offset: 11, Value: payload},
}
processStatBatchReliablyForWorker(
context.Background(),
discardStatLogger{},
registry,
appender,
committer,
quarantiner,
messages,
time.Nanosecond,
nil,
)
if appender.count != 2 {
t.Fatalf("append attempts = %d, want failed message once and following message once", appender.count)
}
if quarantiner.count != 1 || quarantiner.message.Offset != 10 {
t.Fatalf("quarantine = %d/%d, want one message at offset 10", quarantiner.count, quarantiner.message.Offset)
}
if got := committedOffsets(committer.messages); len(got) != 2 || got[0] != 10 || got[1] != 11 {
t.Fatalf("committed offsets = %#v, want [10 11]", got)
}
text := registry.Render()
if !strings.Contains(text, `vehicle_stat_quarantine_total{status="ok",topic="vehicle.fields.go.jt808.v1"} 1`) {
t.Fatalf("quarantine success metric missing:\n%s", text)
}
}
func TestFileStatMessageQuarantinerPersistsOriginalKafkaMessage(t *testing.T) {
dir := t.TempDir()
message := kafka.Message{
Topic: "vehicle.fields.go.jt808.v1",
Partition: 2,
Offset: 5094052,
Key: []byte("vehicle-key"),
Value: []byte(`{"event_id":"poison:fields"}`),
}
wantErr := errors.New("permanent mysql error")
if err := (fileStatMessageQuarantiner{dir: dir}).Quarantine(context.Background(), message, wantErr); err != nil {
t.Fatalf("Quarantine() error = %v", err)
}
payload, err := os.ReadFile(filepath.Join(dir, "vehicle.fields.go.jt808.v1-2-5094052.json"))
if err != nil {
t.Fatalf("read quarantine record: %v", err)
}
var record quarantinedStatMessage
if err := json.Unmarshal(payload, &record); err != nil {
t.Fatalf("unmarshal quarantine record: %v", err)
}
if record.Topic != message.Topic || record.Partition != message.Partition || record.Offset != message.Offset || string(record.Key) != string(message.Key) || string(record.Value) != string(message.Value) || record.Error != wantErr.Error() {
t.Fatalf("quarantine record = %#v, want original message and error", record)
}
}
func TestProcessStatBatchReliablyRetriesCommitWithoutReappending(t *testing.T) {
env := envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808, Phone: "13307795425", MessageID: "0x0200"}
payload := marshalStatTestFieldsEnvelope(t, env)
@@ -826,6 +894,30 @@ func TestRetryStatAppenderRecordsExhaustedTransientError(t *testing.T) {
}
}
func TestRetryStatAppenderRetriesQuarantinableDataErrorBeforeIsolation(t *testing.T) {
registry := metrics.NewRegistry()
delegate := &contextCheckingStatAppender{err: errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null")}
appender := retryStatAppender{delegate: delegate, attempts: 3, registry: registry}
_, err := appender.AppendWithResult(context.Background(), envelope.FrameEnvelope{Protocol: envelope.ProtocolJT808})
if err == nil {
t.Fatal("AppendWithResult() error = nil, want exhausted data error")
}
if delegate.count != 3 {
t.Fatalf("append attempts = %d, want configured finite retries", delegate.count)
}
text := registry.Render()
for _, want := range []string{
`vehicle_stat_write_retries_total{operation="single",status="retry"} 2`,
`vehicle_stat_write_retries_total{operation="single",status="exhausted"} 1`,
} {
if !strings.Contains(text, want) {
t.Fatalf("retry metric missing %s:\n%s", want, text)
}
}
}
func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
wantErr := errors.New("validation failed")
delegate := &contextCheckingStatAppender{err: wantErr}
@@ -841,6 +933,34 @@ func TestRetryStatAppenderDoesNotRetryNonTransientError(t *testing.T) {
}
}
func TestIsQuarantinableStatErrorOnlyAcceptsDeterministicRowDataFailures(t *testing.T) {
for _, err := range []error{
errors.New("Error 1048 (23000): Column 'daily_mileage_km' cannot be null"),
errors.New("Error 1264: Out of range value for column 'daily_mileage_km'"),
errors.New("Error 1265: Data truncated for column 'daily_mileage_km'"),
errors.New("Error 1292: Incorrect datetime value"),
errors.New("Error 1366: Incorrect decimal value"),
errors.New("Error 1406: Data too long for column"),
errors.New("Error 3819: Check constraint is violated"),
} {
if !isQuarantinableStatError(err) {
t.Fatalf("isQuarantinableStatError(%q) = false, want true", err.Error())
}
}
for _, err := range []error{
errors.New("Error 1054: Unknown column 'daily_mileage_km'"),
errors.New("Error 1142: INSERT command denied"),
errors.New("validation failed"),
context.Canceled,
context.DeadlineExceeded,
nil,
} {
if isQuarantinableStatError(err) {
t.Fatalf("isQuarantinableStatError(%v) = true, want false", err)
}
}
}
func TestIsTransientMySQLStatError(t *testing.T) {
for _, err := range []error{
errors.New("dial tcp 127.0.0.1:3306: connection refused"),
@@ -903,6 +1023,9 @@ func TestLoadConfigDefaultsToGoFieldsTopics(t *testing.T) {
if cfg.BaselineMissTTL != time.Minute {
t.Fatalf("BaselineMissTTL = %s, want 1m", cfg.BaselineMissTTL)
}
if cfg.BaselineHitTTL != 5*time.Minute {
t.Fatalf("BaselineHitTTL = %s, want 5m", cfg.BaselineHitTTL)
}
if cfg.RetryAttempts != 3 {
t.Fatalf("RetryAttempts = %d, want 3", cfg.RetryAttempts)
}
@@ -965,6 +1088,7 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
t.Setenv("STATS_CACHE_RETENTION_HOURS", "48")
t.Setenv("STATS_CACHE_CLEANUP_INTERVAL_SECONDS", "300")
t.Setenv("STATS_BASELINE_MISS_TTL_SECONDS", "45")
t.Setenv("STATS_BASELINE_HIT_TTL_SECONDS", "180")
t.Setenv("STATS_CACHE_MAX_ENTRIES", "12345")
cfg := loadConfig()
@@ -984,6 +1108,9 @@ func TestLoadConfigSetsStatWriteIntervals(t *testing.T) {
if cfg.BaselineMissTTL != 45*time.Second {
t.Fatalf("BaselineMissTTL = %s, want 45s", cfg.BaselineMissTTL)
}
if cfg.BaselineHitTTL != 3*time.Minute {
t.Fatalf("BaselineHitTTL = %s, want 3m", cfg.BaselineHitTTL)
}
if cfg.CacheMaxEntries != 12345 {
t.Fatalf("CacheMaxEntries = %d, want 12345", cfg.CacheMaxEntries)
}
@@ -997,6 +1124,7 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
t.Setenv("STATS_RETRY_DELAY_MS", "33")
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_ON_START", "false")
t.Setenv("STATS_NORMALIZE_PLATFORM_SOURCES_TIMEOUT_SECONDS", "17")
t.Setenv("STATS_QUARANTINE_DIR", "/tmp/stat-writer-quarantine-test")
cfg := loadConfig()
@@ -1021,6 +1149,9 @@ func TestLoadConfigReadsStatBatchSettings(t *testing.T) {
if cfg.NormalizePlatformSourcesTimeout != 17*time.Second {
t.Fatalf("NormalizePlatformSourcesTimeout = %s, want 17s", cfg.NormalizePlatformSourcesTimeout)
}
if cfg.QuarantineDir != "/tmp/stat-writer-quarantine-test" {
t.Fatalf("QuarantineDir = %q, want env override", cfg.QuarantineDir)
}
}
type contextCheckingStatAppender struct {
@@ -1070,6 +1201,18 @@ type contextCheckingStatCommitter struct {
failOnCount int
}
type recordingStatQuarantiner struct {
count int
message kafka.Message
err error
}
func (q *recordingStatQuarantiner) Quarantine(_ context.Context, message kafka.Message, _ error) error {
q.count++
q.message = message
return q.err
}
func (c *contextCheckingStatCommitter) CommitMessages(ctx context.Context, messages ...kafka.Message) error {
c.ctxErr = ctx.Err()
c.count++