shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): harden telemetry pipeline and unify Semi UI workspaces

Browse Source
This commit is contained in:
lingniu
2026-07-18 00:26:36 +08:00
parent 65b4e4f055
commit 159c80b0ae
136 changed files with 21616 additions and 1785 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
go/vehicle-gateway/cmd/stats-backfill/main.go
View File

@@ -36,6 +36,7 @@ type config struct {
Debug bool
EventTimeFullScan bool
ProgressEvery int64
BaselineLookback int
Location *time.Location
}
@@ -50,6 +51,7 @@ type rawFrameRow struct {
EventTimeMS int64
ReceivedAtMS int64
ParsedJSON string
RawText string
}
type metricAgg struct {
@@ -188,6 +190,9 @@ func main() {
}
}
fields := fieldsForStats(row.Protocol, row.VIN, text)
if len(fields) == 0 && row.Protocol == envelope.ProtocolYutongMQTT {
fields = fieldsForStats(row.Protocol, row.VIN, row.RawText)
}
if cfg.Debug && scanned <= 5 {
slog.Info("debug raw frame",
"protocol", row.Protocol,
@@ -427,7 +432,7 @@ func buildLastDiffAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB,
return aggregates, nil
}
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, tdDB *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB, _ *sql.DB, cfg config, aggregates map[string]*metricAgg) (int, error) {
if aggregates == nil {
return 0, fmt.Errorf("aggregates map is nil")
}
@@ -441,11 +446,6 @@ func addRealtimeLocationFallbackAggregates(ctx context.Context, mysqlDB *sql.DB,
continue
}
latestHistory := map[sourceHistoryID]dailySourceLast{}
preWindow, err := queryPreviousLastSourceRows(ctx, tdDB, cfg, protocol, targetDates[0])
if err != nil {
return added, err
}
rememberLatestSourceRows(latestHistory, preWindow)
aggregateRowsByDate := indexAggregateSourceRowsByDate(aggregates, protocol)
for _, date := range targetDates {
current, err := queryRealtimeLocationLastRows(ctx, mysqlDB, cfg, protocol, date)
@@ -656,7 +656,9 @@ func queryDailyLastSourceRows(ctx context.Context, db *sql.DB, cfg config, proto
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, FIRST(event_time), FIRST(parsed_json), LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
FIRST(event_time), FIRST(parsed_json), FIRST(raw_text),
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -668,7 +670,7 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
// LAST aggregates the complete pre-window history once per source so empty
// calendar days do not make a backfill fall back to the current day's first
// sample.
where := backfillBeforePredicates(date)
where := backfillBeforePredicates(cfg, date)
where = append(where,
"parse_status = 'OK'",
"vin IS NOT NULL",
@@ -679,17 +681,31 @@ func queryPreviousLastSourceRows(ctx context.Context, db *sql.DB, cfg config, pr
if predicate := mileageBearingFramePredicate(protocol); predicate != "" {
where = append(where, predicate)
}
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint, LAST(event_time), LAST(parsed_json), COUNT(*)
sqlText := fmt.Sprintf(`SELECT vin, phone, device_id, source_endpoint,
LAST(event_time), LAST(parsed_json), LAST(raw_text), COUNT(*)
FROM %s.raw_frames
WHERE %s
GROUP BY vin, phone, device_id, source_endpoint`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
return querySourceRows(ctx, db, cfg, protocol, sqlText, false)
}
func backfillBeforePredicates(eventDateExclusive string) []string {
return []string{
func backfillBeforePredicates(cfg config, eventDateExclusive string) []string {
lookbackDays := cfg.BaselineLookback
if lookbackDays <= 0 {
lookbackDays = 7
}
eventDateFrom := shiftDate(eventDateExclusive, -lookbackDays)
where := []string{
fmt.Sprintf("event_time >= '%s 00:00:00'", quote(eventDateFrom)),
fmt.Sprintf("event_time < '%s 00:00:00'", quote(eventDateExclusive)),
}
if !cfg.EventTimeFullScan {
where = append([]string{
fmt.Sprintf("ts >= '%s 00:00:00'", quote(previousDate(eventDateFrom))),
fmt.Sprintf("ts < '%s 00:00:00'", quote(nextDate(eventDateExclusive))),
}, where...)
}
return where
}
func queryRealtimeLocationLastRows(ctx context.Context, db *sql.DB, cfg config, protocol envelope.Protocol, date string) (map[string][]dailySourceLast, error) {
@@ -775,27 +791,30 @@ func querySourceRows(ctx context.Context, db *sql.DB, cfg config, protocol envel
var sourceEndpoint string
var firstTS time.Time
var firstParsedJSON string
var firstRawText string
var ts time.Time
var parsedJSON string
var rawText string
var rawSampleCount int64
if includeFirst {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &firstTS, &firstParsedJSON, &firstRawText, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
} else {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawSampleCount); err != nil {
if err := rows.Scan(&vin, &phone, &deviceID, &sourceEndpoint, &ts, &parsedJSON, &rawText, &rawSampleCount); err != nil {
return nil, err
}
firstTS = ts
firstParsedJSON = parsedJSON
firstRawText = rawText
}
latestTotalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, ts, cfg.Location)
latestTotalKM, ok := mileageFromEvidence(protocol, vin, parsedJSON, rawText, ts, cfg.Location)
if !ok {
continue
}
firstTotalKM := latestTotalKM
if includeFirst {
if parsedFirst, ok := mileageFromParsed(protocol, vin, firstParsedJSON, firstTS, cfg.Location); ok {
if parsedFirst, ok := mileageFromEvidence(protocol, vin, firstParsedJSON, firstRawText, firstTS, cfg.Location); ok {
firstTotalKM = parsedFirst
}
}
@@ -852,6 +871,16 @@ func mileageFromParsed(protocol envelope.Protocol, vin string, parsedJSON string
return samples[0].TotalMileageKM, true
}
func mileageFromEvidence(protocol envelope.Protocol, vin string, parsedJSON string, rawText string, eventTime time.Time, loc *time.Location) (float64, bool) {
if totalKM, ok := mileageFromParsed(protocol, vin, parsedJSON, eventTime, loc); ok {
return totalKM, true
}
if protocol != envelope.ProtocolYutongMQTT || strings.TrimSpace(rawText) == "" {
return 0, false
}
return mileageFromParsed(protocol, vin, rawText, eventTime, loc)
}
func normalizedSourceKey(protocol string, phone string, deviceID string, endpoint string) string {
return normalizedSourceKeyForSource(protocol, phone, deviceID, endpoint, "", "")
}
@@ -951,6 +980,7 @@ func loadConfig() (config, error) {
Debug: envBool("BACKFILL_DEBUG", false),
EventTimeFullScan: envBool("BACKFILL_EVENT_TIME_FULL_SCAN", false),
ProgressEvery: int64(envInt("BACKFILL_PROGRESS_EVERY", 100000)),
BaselineLookback: envInt("BACKFILL_BASELINE_LOOKBACK_DAYS", 7),
Location: loc,
}, nil
}
@@ -1000,7 +1030,7 @@ func queryRawFrames(ctx context.Context, db *sql.DB, cfg config) (*sql.Rows, err
where = append(where, "protocol IN ("+strings.Join(quoted, ",")+")")
}
where = append(where, realtimeMileageFramePredicate())
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json
sqlText := fmt.Sprintf(`SELECT protocol, vin, phone, device_id, source_endpoint, event_id, message_id, event_time, received_at, parsed_json, raw_text
FROM %s.raw_frames
WHERE %s
ORDER BY event_time ASC, ts ASC`, ident(cfg.TDengineDatabase), strings.Join(where, " AND "))
@@ -1044,6 +1074,9 @@ func mileageBearingFramePredicate(protocol envelope.Protocol) string {
conditions := make([]string, 0, len(tokens))
for _, token := range tokens {
conditions = append(conditions, fmt.Sprintf("parsed_json LIKE '%%%s%%'", quote(token)))
if protocol == envelope.ProtocolYutongMQTT {
conditions = append(conditions, fmt.Sprintf("raw_text LIKE '%%%s%%'", quote(token)))
}
}
return "(" + strings.Join(conditions, " OR ") + ")"
}
@@ -1071,10 +1104,10 @@ func mileageSearchTokens(protocol envelope.Protocol) []string {
}
func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed string
var protocol, vin, phone, deviceID, sourceEndpoint, eventID, parsed, rawText string
var messageID int64
var eventTime, receivedAt time.Time
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed); err != nil {
if err := rows.Scan(&protocol, &vin, &phone, &deviceID, &sourceEndpoint, &eventID, &messageID, &eventTime, &receivedAt, &parsed, &rawText); err != nil {
return rawFrameRow{}, err
}
return rawFrameRow{
@@ -1088,6 +1121,7 @@ func scanRawFrame(rows *sql.Rows) (rawFrameRow, error) {
EventTimeMS: eventTime.UnixMilli(),
ReceivedAtMS: receivedAt.UnixMilli(),
ParsedJSON: parsed,
RawText: rawText,
}, nil
}
@@ -1267,6 +1301,14 @@ func previousDate(date string) string {
return parsed.AddDate(0, 0, -1).Format("2006-01-02")
}
func shiftDate(date string, days int) string {
parsed, err := time.Parse("2006-01-02", date)
if err != nil {
return date
}
return parsed.AddDate(0, 0, days).Format("2006-01-02")
}
func dateRangeWithPrevious(from string, to string) ([]string, error) {
dates, err := dateRange(from, to)
if err != nil {