shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: build vehicle data platform and production pipeline

Browse Source
This commit is contained in:
lingniu
2026-07-14 12:35:33 +08:00
parent b452be3b94
commit bb59303a4b
270 changed files with 88016 additions and 1975 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go
View File

@@ -33,6 +33,9 @@ func buildRawFrameSQL(database string, query RawFrameQuery) SQLQuery {
if len(where) > 0 {
countText += ` WHERE ` + strings.Join(where, " AND ")
}
if query.SkipCount {
countText = ""
}
text += ` ORDER BY ts DESC, frame_id ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args, CountText: countText}
}
@@ -71,7 +74,7 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery
where = append(where, "ts <= '"+quoteTDengine(normalizeTDengineTime(dateTo))+"'")
}
limit, offset := parseLimitOffset(query["limit"], query["offset"])
text := `SELECT ts, vin, protocol, longitude, latitude, speed_kmh, total_mileage_km, received_at FROM ` + table
text := `SELECT CAST(ts AS BIGINT), vin, protocol, longitude, latitude, speed_kmh, soc_percent, direction_deg, alarm_flag, total_mileage_km, CAST(received_at AS BIGINT) FROM ` + table
if len(where) > 0 {
text += ` WHERE ` + strings.Join(where, " AND ")
}
@@ -83,6 +86,90 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery
return SQLQuery{Text: text, Args: args, CountText: countText}
}
func buildHistoryLocationSeriesSQL(database string, query HistoryLocationSeriesQuery) SQLQuery {
tableQuery := map[string]string{"vin": query.VIN, "protocol": query.Protocol}
where := []string{"vin = '" + quoteTDengine(strings.TrimSpace(query.VIN)) + "'"}
if protocol := strings.TrimSpace(query.Protocol); protocol != "" {
where = append(where, "protocol = '"+quoteTDengine(strings.ToUpper(protocol))+"'")
}
where = append(where,
"ts >= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateFrom))+"'",
"ts <= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateTo))+"'",
)
grain := query.GrainSeconds
if grain <= 0 {
grain = 60
}
text := `SELECT _wstart, protocol, COUNT(*), ` +
`AVG(speed_kmh), MIN(speed_kmh), MAX(speed_kmh), LAST(speed_kmh), ` +
`AVG(total_mileage_km), MIN(total_mileage_km), MAX(total_mileage_km), LAST(total_mileage_km) ` +
`FROM ` + locationTable(database, tableQuery) + ` WHERE ` + strings.Join(where, " AND ") +
` PARTITION BY protocol INTERVAL(` + strconv.Itoa(grain) + `s) ORDER BY _wstart ASC, protocol ASC`
return SQLQuery{Text: text}
}
func buildHistoryExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery {
if limit <= 0 || limit > 5000 {
limit = 5000
}
switch query.Category {
case "raw":
return buildRawExportBatchSQL(database, query, cursor, limit)
default:
return buildLocationExportBatchSQL(database, query, cursor, limit)
}
}
func buildLocationExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery {
tableQuery := map[string]string{"vin": query.VIN, "protocol": query.Protocol}
where := historyExportTDengineWhere(query)
if cursor.Time != "" {
cursorTime := quoteTDengine(cursor.Time)
cursorProtocol := quoteTDengine(cursor.Protocol)
where = append(where, "(ts > '"+cursorTime+"' OR (ts = '"+cursorTime+"' AND protocol > '"+cursorProtocol+"'))")
}
table := locationTable(database, tableQuery)
return SQLQuery{
Text: `SELECT ts, vin, protocol, longitude, latitude, speed_kmh, total_mileage_km, received_at FROM ` + table + ` WHERE ` + strings.Join(where, " AND ") + ` ORDER BY ts ASC, protocol ASC LIMIT ` + strconv.Itoa(limit),
CountText: `SELECT COUNT(*) FROM ` + table + ` WHERE ` + strings.Join(historyExportTDengineWhere(query), " AND "),
}
}
func buildRawExportBatchSQL(database string, query HistoryExportStoreQuery, cursor HistoryExportCursor, limit int) SQLQuery {
rawQuery := RawFrameQuery{VIN: query.VIN, Protocol: query.Protocol}
where := historyExportTDengineWhere(query)
if cursor.Time != "" {
timeValue := quoteTDengine(cursor.Time)
protocol := quoteTDengine(cursor.Protocol)
identifier := quoteTDengine(cursor.ID)
where = append(where, "(ts > '"+timeValue+"' OR (ts = '"+timeValue+"' AND (protocol > '"+protocol+"' OR (protocol = '"+protocol+"' AND frame_id > '"+identifier+"'))))")
}
table := rawFrameTable(database, rawQuery)
selectText := `SELECT ts, frame_id, event_time, received_at, raw_size_bytes, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone FROM ` + table + ` WHERE ` + strings.Join(where, " AND ")
return SQLQuery{Text: selectText + ` ORDER BY ts ASC, protocol ASC, frame_id ASC LIMIT ` + strconv.Itoa(limit), CountText: `SELECT COUNT(*) FROM ` + table + ` WHERE ` + strings.Join(historyExportTDengineWhere(query), " AND ")}
}
func historyExportTDengineWhere(query HistoryExportStoreQuery) []string {
where := []string{"vin = '" + quoteTDengine(strings.TrimSpace(query.VIN)) + "'"}
if protocol := strings.TrimSpace(query.Protocol); protocol != "" {
where = append(where, "protocol = '"+quoteTDengine(strings.ToUpper(protocol))+"'")
}
if query.DateFrom != "" {
where = append(where, "ts >= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateFrom))+"'")
}
if query.DateTo != "" {
where = append(where, "ts <= '"+quoteTDengine(normalizeTDengineSeriesTime(query.DateTo))+"'")
}
return where
}
func normalizeTDengineSeriesTime(value string) string {
if parsed, ok := parseTrackRequestTime(value); ok {
return parsed.Format(time.RFC3339)
}
return strings.TrimSpace(value)
}
func buildTodayRawFrameCountSQL(database string, now time.Time) SQLQuery {
start := shanghaiDayStartUTC(now)
return SQLQuery{
@@ -133,13 +220,13 @@ func normalizeTDengineTime(value string) string {
return ""
}
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02 15:04:05", "2006-01-02"} {
for _, layout := range []string{"2006-01-02T15:04:05", "2006-01-02T15:04", "2006-01-02 15:04:05", "2006-01-02 15:04", "2006-01-02"} {
if parsed, err := time.ParseInLocation(layout, value, shanghai); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
return parsed.Format(time.RFC3339)
}
}
if parsed, err := time.Parse(time.RFC3339, value); err == nil {
return parsed.UTC().Format("2006-01-02 15:04:05")
return parsed.Format(time.RFC3339)
}
return value
}
@@ -148,5 +235,5 @@ func shanghaiDayStartUTC(now time.Time) string {
shanghai := time.FixedZone("Asia/Shanghai", 8*3600)
local := now.In(shanghai)
start := time.Date(local.Year(), local.Month(), local.Day(), 0, 0, 0, 0, shanghai)
return start.UTC().Format("2006-01-02 15:04:05")
return start.Format(time.RFC3339)
}