shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): harden telemetry pipeline and unify Semi UI workspaces

Browse Source
This commit is contained in:
lingniu
2026-07-18 00:26:36 +08:00
parent 65b4e4f055
commit 159c80b0ae
136 changed files with 21616 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
go/vehicle-gateway/internal/stats/daily_metric.go
View File

@@ -18,6 +18,7 @@ const (
defaultCacheRetention = 72 * time.Hour
defaultCacheCleanupInterval = 10 * time.Minute
defaultBaselineMissTTL = time.Minute
defaultBaselineHitTTL = 5 * time.Minute
defaultMaxCacheEntries = 1000000
)
@@ -34,6 +35,7 @@ type Writer struct {
cacheRetention time.Duration
cacheCleanupInterval time.Duration
baselineMissTTL time.Duration
baselineHitTTL time.Duration
maxCacheEntries int
lastCacheCleanup time.Time
lastCacheCleanupStats cacheCleanupStats
@@ -122,6 +124,7 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
cacheRetention: defaultCacheRetention,
cacheCleanupInterval: defaultCacheCleanupInterval,
baselineMissTTL: defaultBaselineMissTTL,
baselineHitTTL: defaultBaselineHitTTL,
maxCacheEntries: defaultMaxCacheEntries,
lastTotalMileage: map[string]float64{},
lastSourceSeen: map[string]time.Time{},
@@ -182,6 +185,15 @@ func (w *Writer) SetBaselineMissTTL(ttl time.Duration) {
w.baselineMissTTL = ttl
}
func (w *Writer) SetBaselineHitTTL(ttl time.Duration) {
w.mu.Lock()
defer w.mu.Unlock()
if ttl < 0 {
ttl = 0
}
w.baselineHitTTL = ttl
}
func (w *Writer) SetMaxCacheEntries(maxEntries int) {
w.mu.Lock()
defer w.mu.Unlock()
@@ -517,8 +529,12 @@ func (w *Writer) previousBaseline(ctx context.Context, candidate SourceMileageSa
now := time.Now()
w.mu.Lock()
if cached, ok := w.baselineCache[cacheKey]; ok {
missExpired := !cached.found && (w.baselineMissTTL == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= w.baselineMissTTL)
if !missExpired {
ttl := w.baselineMissTTL
if cached.found {
ttl = w.baselineHitTTL
}
expired := ttl == 0 || cached.cachedAt.IsZero() || now.Sub(cached.cachedAt) >= ttl
if !expired {
w.mu.Unlock()
return cached.baseline, cached.found, nil
}
@@ -570,6 +586,16 @@ func (w *Writer) cacheBaseline(candidate SourceMileageSample, cacheKey string, e
return
}
w.mu.Lock()
if previous, ok := w.baselineCache[cacheKey]; ok &&
previous.found && entry.found &&
previous.baseline.LatestTotalKM == entry.baseline.LatestTotalKM &&
previous.baseline.LatestEventTime.Equal(entry.baseline.LatestEventTime) &&
!previous.cachedAt.IsZero() {
// markBaselineWritten runs for every accepted sample. Preserve the
// original cache age when the durable day-boundary baseline has not
// changed, otherwise an active vehicle would keep stale history forever.
entry.cachedAt = previous.cachedAt
}
if entry.cachedAt.IsZero() {
entry.cachedAt = time.Now()
}

85
go/vehicle-gateway/internal/stats/daily_metric_test.go
View File

@@ -1091,6 +1091,91 @@ func TestWriterRetriesExpiredMissingPreviousDayBaseline(t *testing.T) {
}
}
func TestWriterRefreshesExpiredFoundPreviousDayBaseline(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
writer.SetBaselineHitTTL(time.Minute)
candidate := SourceMileageSample{
VIN: "LMRKH9AC6R1004111",
StatDate: "2026-07-17",
Protocol: envelope.ProtocolYutongMQTT,
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
}
cacheKey := mileageCacheKey(MetricSample{
VIN: candidate.VIN,
Protocol: candidate.Protocol,
StatDate: candidate.StatDate,
SourceKey: candidate.SourceKey,
})
staleTime := time.Date(2026, 7, 12, 4, 12, 20, 0, loc)
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
baseline: sourceBaseline{LatestTotalKM: 90778, LatestEventTime: staleTime},
found: true,
cachedAt: time.Now().Add(-2 * time.Minute),
}
refreshedTime := time.Date(2026, 7, 16, 23, 59, 59, 0, loc)
mock.ExpectQuery(`SELECT latest_total_mileage_km, latest_event_time FROM vehicle_daily_mileage_source`).
WithArgs(candidate.VIN, candidate.StatDate, "YUTONG_MQTT", candidate.SourceKey).
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).
AddRow(91302.0, refreshedTime))
baseline, found, err := writer.previousBaseline(context.Background(), candidate)
if err != nil || !found {
t.Fatalf("previousBaseline() found=%v error=%v, want refreshed baseline", found, err)
}
if baseline.LatestTotalKM != 91302 || !baseline.LatestEventTime.Equal(refreshedTime) {
t.Fatalf("baseline = %+v, want nearest refreshed baseline", baseline)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestWriterMarkBaselineKeepsCacheAgeForUnchangedBoundary(t *testing.T) {
db, _, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
loc := time.FixedZone("Asia/Shanghai", 8*3600)
writer := NewWriter(db, loc)
candidate := SourceMileageSample{
VIN: "LMRKH9AC6R1004111",
StatDate: "2026-07-17",
Protocol: envelope.ProtocolYutongMQTT,
SourceKey: "YUTONG_MQTT:LMRKH9AC6R1004111@PLATFORM:yutong",
FirstTotalKM: 91302,
FirstEventTime: time.Date(2026, 7, 16, 23, 59, 59, 0, loc),
}
sample := MetricSample{
VIN: candidate.VIN,
Protocol: candidate.Protocol,
StatDate: candidate.StatDate,
SourceKey: candidate.SourceKey,
}
cacheKey := mileageCacheKey(sample)
cachedAt := time.Now().Add(-4 * time.Minute)
writer.baselineCache[cacheKey] = sourceBaselineCacheEntry{
baseline: sourceBaseline{LatestTotalKM: candidate.FirstTotalKM, LatestEventTime: candidate.FirstEventTime},
found: true,
cachedAt: cachedAt,
}
writer.markBaselineWritten(sample, candidate)
if got := writer.baselineCache[cacheKey].cachedAt; !got.Equal(cachedAt) {
t.Fatalf("cachedAt = %s, want unchanged %s", got, cachedAt)
}
}
func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {

28
go/vehicle-gateway/internal/stats/source_mileage.go
View File

@@ -337,10 +337,30 @@ func projectDailyMileageWithExec(ctx context.Context, exec Execer, vin string, s
return err
}
const upsertSourceStatDateStartSQL = `CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)`
const upsertSourcePreferIncomingFirstSQL = `(
first_event_time IS NULL
OR (
first_event_time < ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) >= first_event_time
)
OR (
first_event_time >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) < ` + upsertSourceStatDateStartSQL + `
)
OR (
first_event_time >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) >= ` + upsertSourceStatDateStartSQL + `
AND VALUES(first_event_time) <= first_event_time
)
)`
const upsertSourceMergedFirstTotalSQL = `CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
THEN VALUES(first_total_mileage_km)
ELSE first_total_mileage_km
END`
@@ -354,7 +374,7 @@ const upsertSourceMergedLatestTotalSQL = `CASE
END`
const upsertSourceMergedFirstEventSQL = `CASE
WHEN first_event_time IS NULL OR VALUES(first_event_time) <= first_event_time
WHEN ` + upsertSourcePreferIncomingFirstSQL + `
THEN VALUES(first_event_time)
ELSE first_event_time
END`
@@ -490,6 +510,8 @@ WHERE s.vin = ?
AND s.stat_date = ?
AND s.protocol = ?
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
AND s.first_total_mileage_km IS NOT NULL
AND s.latest_total_mileage_km IS NOT NULL
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))
@@ -558,6 +580,8 @@ WHERE s.vin = ?
AND s.stat_date = ?
AND s.protocol = ?
AND s.quality_status IN ('` + QualityOK + `', '` + QualityNoPreviousBaseline + `')
AND s.first_total_mileage_km IS NOT NULL
AND s.latest_total_mileage_km IS NOT NULL
AND COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code) IS NOT NULL
AND COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')) IS NOT NULL
AND s.source_key <> CONCAT(s.protocol, ':', COALESCE(NULLIF(TRIM(s.phone), ''), NULLIF(TRIM(s.device_id), '')), '` + platformSourceKeyPrefix + `', COALESCE(NULLIF(TRIM(ds.source_code), ''), vi.source_code))

18
go/vehicle-gateway/internal/stats/source_mileage_test.go
View File

@@ -187,7 +187,9 @@ func TestUpsertSourceMileageTruncatesSubsecondEventTimesAtDayBoundary(t *testing
func TestUpsertSourceMileageUsesEventTimeBoundaries(t *testing.T) {
for _, want := range []string{
"first_total_mileage_km = CASE",
"VALUES(first_event_time) >= first_event_time",
"VALUES(first_event_time) <= first_event_time",
"VALUES(first_event_time) < CAST(CONCAT(VALUES(stat_date), ' 00:00:00') AS DATETIME)",
"latest_total_mileage_km = CASE",
"VALUES(latest_event_time) >= latest_event_time",
"daily_mileage_km = CASE",
@@ -334,6 +336,22 @@ func TestNormalizePlatformSourceMileageMergesLegacyIPKeys(t *testing.T) {
}
}
func TestNormalizePlatformSourceMileageExcludesIncompleteLegacyTotals(t *testing.T) {
for name, query := range map[string]string{
"insert": normalizePlatformSourceMileageInsertSQL,
"delete": normalizePlatformSourceMileageDeleteSQL,
} {
for _, predicate := range []string{
"s.first_total_mileage_km IS NOT NULL",
"s.latest_total_mileage_km IS NOT NULL",
} {
if !strings.Contains(query, predicate) {
t.Fatalf("%s query missing incomplete-total guard %q:\n%s", name, predicate, query)
}
}
}
}
func TestNormalizePlatformSourceMileageForDateNormalizesAndProjectsLegacyVINs(t *testing.T) {
db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))
if err != nil {