shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): harden telemetry pipeline and unify Semi UI workspaces

Browse Source
This commit is contained in:
lingniu
2026-07-18 00:26:36 +08:00
parent 65b4e4f055
commit 159c80b0ae
136 changed files with 21616 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -36,6 +36,7 @@ type config struct {
Debug bool
EventTimeFullScan bool
ProgressEvery int64
BaselineLookback int
Location *time.Location
}
@@ -50,6 +51,7 @@ type rawFrameRow struct {
EventTimeMS int64
ReceivedAtMS int64
ParsedJSON string
RawText string
}
type metricAgg struct {
@@ -188,6 +190,9 @@ func main() {
}
}
fields := fieldsForStats(row.Protocol, row.VIN, text)
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
}
if cfg.Debug && scanned <= 5 {
slog.Info("debug raw frame",
"protocol", row.Protocol,
@@ -427,7 +432,7 @@ func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB,
return aggregates, nil
}
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, _ *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if aggregates == nil {
return 0, fmt.Errorf("aggregates map is nil")
}
@@ -441,11 +446,6 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
continue
}
latestHistory := map[sourceHistoryID]dailySourceLast{}
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
if err != nil {
return added, err
}
rememberLatestSourceRows(latestHistory, preWindow)
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
for _, date := range targetDates {
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
@@ -656,7 +656,9 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -668,7 +670,7 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
// LAST aggregates the complete pre-window history once per source so empty
// calendar days do not make a backfill fall back to the current day's first
// sample.
where := backfillBeforePredicates(date)
where := backfillBeforePredicates(cfg, date)
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
@@ -679,17 +681,31 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
}
func backfillBeforePredicates(eventDateExclusive string) []string {
return []string{
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
lookbackDays := cfg.BaselineLookback
if lookbackDays <= 0 {
lookbackDays = 7
}
eventDateFrom := shiftDate(eventDateExclusive, -lookbackDays)
where := []string{
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
}
if !cfg.EventTimeFullScan {
where = append([]string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateExclusive))),
}, where...)
}
return where
}
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
@@ -775,27 +791,30 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
var sourceEndpoint string
var firstTS time.Time
var firstParsedJSON string
var firstRawText string
var ts time.Time
var parsedJSON string
var rawText string
var rawSampleCount int64
if includeFirst {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
} else {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
firstTS = ts
firstParsedJSON = parsedJSON
firstRawText = rawText
}
latestTotalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, ts, cfg.Location)
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
if !ok {
continue
}
firstTotalKM := latestTotalKM
if includeFirst {
if parsedFirst, ok := mileageFromParsed(protocol, vin, firstParsedJSON, firstTS, cfg.Location); ok {
if parsedFirst, ok := mileageFromEvidence(protocol, vin, firstParsedJSON, firstRawText, firstTS, cfg.Location); ok {
firstTotalKM = parsedFirst
}
}
@@ -852,6 +871,16 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
return samples[0].TotalMileageKM, true
}
func mileageFromEvidence(protocol envelope.Protocol, vin string, parsedJSON string, rawText string, eventTime time.Time, loc *time.Location) (float64, bool) {
if totalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, eventTime, loc); ok {
return totalKM, true
}
if protocol != envelope.ProtocolYutongMQTT || strings.TrimSpace(rawText) == "" {
return 0, false
}
return mileageFromParsed(protocol, vin, rawText, eventTime, loc)
}
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
}
@@ -951,6 +980,7 @@ func loadConfig() (config, error) {
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
Location: loc,
}, nil
}
@@ -1000,7 +1030,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
}
where = append(where, realtimeMileageFramePredicate())
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json, raw_text
FROM %s.raw_frames
WHERE %s
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -1044,6 +1074,9 @@ func mileageBearingFramePredicate(protocol envelope.Protocol) string {
conditions := make([]string, 0, len(tokens))
for _, token := range tokens {
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
if protocol == envelope.ProtocolYutongMQTT {
conditions = append(conditions, fmt.Sprintf("raw_text LIKE '%%%s%%'", quote(token)))
}
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
@@ -1071,10 +1104,10 @@ func mileageSearchTokens(protocol envelope.Protocol) []string {
}
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed, rawText string
var messageID int64
var eventTime, receivedAt time.Time
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed); err != nil {
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed, &rawText); err != nil {
return rawFrameRow{}, err
}
return rawFrameRow{
@@ -1088,6 +1121,7 @@ func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
EventTimeMS: eventTime.UnixMilli(),
ReceivedAtMS: receivedAt.UnixMilli(),
ParsedJSON: parsed,
RawText: rawText,
}, nil
}
@@ -1267,6 +1301,14 @@ func previousDate(date string) string {
return parsed.AddDate(0, 0, -1).Format("2006-01-02")
}
func shiftDate(date string, days int) string {
parsed, err := time.Parse("2006-01-02", date)
if err != nil {
return date
}
return parsed.AddDate(0, 0, days).Format("2006-01-02")
}
func dateRangeWithPrevious(from string, to string) ([]string, error) {
dates, err := dateRange(from, to)
if err != nil {

60
go/vehicle-gateway/cmd/stats-backfill/main_test.go
View File

@@ -233,23 +233,23 @@ func TestBuildLastDiffAggregatesCarriesNearestHistoryAcrossEmptyDays(t *testing.
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
currentRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
}
previousRows := func() *sqlmock.Rows {
return sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
})
}
mock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(previousRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, int64(4)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", dayOneTS, `{"data":{"TOTAL_MILEAGE":100000}}`, "", int64(4)))
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-10 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows())
mock.ExpectQuery("(?s)SELECT vin.*FIRST\\(event_time\\).*event_time >= '2026-07-11 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, int64(6)))
WillReturnRows(currentRows().AddRow(vin, "", vin, endpoint, dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", dayThreeTS, `{"data":{"TOTAL_MILEAGE":120000}}`, "", int64(6)))
aggregates, err := buildLastDiffAggregates(context.Background(), nil, tdDB, config{
TDengineDatabase: "lingniu_vehicle_ts",
@@ -342,18 +342,10 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
WithArgs("YUTONG_MQTT", "2026-07-12", "2026-07-12").
WillReturnRows(sqlmock.NewRows([]string{"vin", "peer", "total_mileage_event_time", "total_mileage_km"}).
AddRow("LMRKH9AC0R1004086", "mqtt://yutong/ytforward/shln/4", currentTS, 11578.0))
tdMock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-12 00:00:00'.*protocol = 'YUTONG_MQTT'.*TOTAL_MILEAGE").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}).AddRow(
"LMRKH9AC0R1004086",
"",
"LMRKH9AC0R1004086",
"mqtt://yutong/ytforward/shln/4",
previousTS,
`{"data":{"TOTAL_MILEAGE":11500000}}`,
int64(12),
))
sourceKey := stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong")
mysqlMock.ExpectQuery("(?s)SELECT latest_total_mileage_km, latest_event_time.*FROM vehicle_daily_mileage_source").
WithArgs("LMRKH9AC0R1004086", "2026-07-12", "YUTONG_MQTT", sourceKey).
WillReturnRows(sqlmock.NewRows([]string{"latest_total_mileage_km", "latest_event_time"}).AddRow(11500.0, previousTS))
aggregates := map[string]*metricAgg{}
added, err := addRealtimeLocationFallbackAggregates(context.Background(), mysqlDB, tdDB, config{
@@ -373,7 +365,7 @@ func TestAddRealtimeLocationFallbackAggregatesUsesHistoricalBaseline(t *testing.
if agg.FirstKM != 11500 || agg.LatestKM != 11578 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.SourceKey != stats.SourceKeyForSource(envelope.ProtocolYutongMQTT, "", "LMRKH9AC0R1004086", "mqtt", "PLATFORM", "yutong") {
if agg.SourceKey != sourceKey {
t.Fatalf("source key = %q", agg.SourceKey)
}
if agg.SourceCode != "yutong" || agg.PlatformName != "宇通" || agg.SourceKind != "PLATFORM" {
@@ -410,10 +402,6 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes
dayOneTS := time.Date(2026, 7, 9, 23, 50, 0, 0, loc)
dayThreeTS := time.Date(2026, 7, 11, 17, 30, 0, 0, loc)
tdMock.ExpectQuery("(?s)SELECT vin.*LAST\\(event_time\\).*event_time < '2026-07-09 00:00:00'.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
}))
for _, date := range []string{"2026-07-09", "2026-07-10"} {
mysqlMock.ExpectQuery("(?s)FROM vehicle_realtime_location l.*l.protocol = \\?").
WithArgs("YUTONG_MQTT", date, date).
@@ -483,16 +471,18 @@ func TestQueryDailyLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 8, 18, 0, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "FIRST(event_time)", "FIRST(parsed_json)", "FIRST(raw_text)", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
firstTS,
`{"yutong_mqtt.data.total_mileage":"65422000"}`,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65422000}}`,
lastTS,
`{"yutong_mqtt.data.total_mileage":"65423000"}`,
`{"yutong_mqtt.data.latitude":"30.1"}`,
`{"data":{"TOTAL_MILEAGE":65423000}}`,
int64(42),
))
@@ -526,14 +516,15 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
lastTS := time.Date(2026, 7, 7, 23, 58, 0, 0, loc)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*event_time < '2026-07-08 00:00:00'.*protocol = 'YUTONG_MQTT'.*parsed_json LIKE '%yutong_mqtt\\.data\\.total_mileage%'.*parsed_json LIKE '%TOTAL_MILEAGE%'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC6R1004108",
"",
"LMRKH9AC6R1004108",
"mqtt://yutong/ytforward/shln/3",
lastTS,
`{"yutong_mqtt.data.total_mileage":"65377000"}`,
`{"yutong_mqtt.data.latitude":"30.0"}`,
`{"data":{"TOTAL_MILEAGE":65377000}}`,
int64(31),
))
@@ -556,13 +547,19 @@ func TestQueryPreviousLastSourceRowsFiltersYutongToMileageFrames(t *testing.T) {
}
}
func TestBackfillBeforePredicatesSearchesAllEarlierHistory(t *testing.T) {
where := strings.Join(backfillBeforePredicates("2026-07-08"), " AND ")
func TestBackfillBeforePredicatesBoundsHistoryScan(t *testing.T) {
where := strings.Join(backfillBeforePredicates(config{BaselineLookback: 7}, "2026-07-08"), " AND ")
if !strings.Contains(where, "event_time < '2026-07-08 00:00:00'") {
t.Fatalf("pre-window predicate missing exclusive upper bound: %s", where)
}
if strings.Contains(where, "event_time >=") || strings.Contains(where, "ts >=") {
t.Fatalf("pre-window predicate must not stop at the previous day: %s", where)
for _, predicate := range []string{
"event_time >= '2026-07-01 00:00:00'",
"ts >= '2026-06-30 00:00:00'",
"ts < '2026-07-09 00:00:00'",
} {
if !strings.Contains(where, predicate) {
t.Fatalf("pre-window predicate missing %q: %s", predicate, where)
}
}
}
@@ -577,7 +574,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
utcInstant := time.Date(2026, 7, 12, 15, 59, 59, 0, time.UTC)
mock.ExpectQuery("(?s)FROM lingniu_vehicle_ts\\.raw_frames.*protocol = 'YUTONG_MQTT'").
WillReturnRows(sqlmock.NewRows([]string{
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "COUNT(*)",
"vin", "phone", "device_id", "source_endpoint", "LAST(event_time)", "LAST(parsed_json)", "LAST(raw_text)", "COUNT(*)",
}).AddRow(
"LMRKH9AC7R1004098",
"",
@@ -585,6 +582,7 @@ func TestQueryPreviousLastSourceRowsNormalizesScannedTimestampToBusinessTimezone
"mqtt://yutong/ytforward/shln/4",
utcInstant,
`{"yutong_mqtt.data.total_mileage":"41249000"}`,
"",
int64(1),
))