shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): paginate history queries

Browse Source
This commit is contained in:
lingniu
2026-07-03 22:19:14 +08:00
parent ea96b6909a
commit 72aa20b347
4 changed files with 91 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
vehicle-data-platform/apps/api/internal/platform/production_store.go
View File

@@ -183,6 +183,12 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer
tdQuery["dateTo"] = value
}
built := buildHistoryLocationSQL(s.tdDatabase, tdQuery)
total := 0
if built.CountText != "" {
if err := s.tdengine.QueryRowContext(ctx, built.CountText, built.CountArgs...).Scan(&total); err != nil {
return Page[HistoryLocationRow]{}, err
}
}
rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...)
if err != nil {
return Page[HistoryLocationRow]{}, err
@@ -207,7 +213,10 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer
if err := rows.Err(); err != nil {
return Page[HistoryLocationRow]{}, err
}
return Page[HistoryLocationRow]{Items: items, Total: len(items), Limit: limit, Offset: offset}, nil
if built.CountText == "" {
total = len(items)
}
return Page[HistoryLocationRow]{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (Page[RawFrameRow], error) {
@@ -215,6 +224,12 @@ func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (P
return Page[RawFrameRow]{Items: []RawFrameRow{}, Total: 0, Limit: query.Limit, Offset: query.Offset}, nil
}
built := buildRawFrameSQL(s.tdDatabase, query)
total := 0
if built.CountText != "" {
if err := s.tdengine.QueryRowContext(ctx, built.CountText, built.CountArgs...).Scan(&total); err != nil {
return Page[RawFrameRow]{}, err
}
}
rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...)
if err != nil {
return Page[RawFrameRow]{}, err
@@ -244,7 +259,10 @@ func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (P
if err := rows.Err(); err != nil {
return Page[RawFrameRow]{}, err
}
return Page[RawFrameRow]{Items: items, Total: len(items), Limit: query.Limit, Offset: query.Offset}, nil
if built.CountText == "" {
total = len(items)
}
return Page[RawFrameRow]{Items: items, Total: total, Limit: query.Limit, Offset: query.Offset}, nil
}
func (s *ProductionStore) DailyMileage(ctx context.Context, query url.Values) (Page[DailyMileageRow], error) {

23
vehicle-data-platform/apps/api/internal/platform/query_builders_test.go
View File

@@ -80,7 +80,28 @@ func TestBuildRawFrameSQL(t *testing.T) {
if !strings.Contains(built.Text, "lingniu_vehicle_ts.raw_gb32960_") || !strings.Contains(built.Text, "parsed_fields") {
t.Fatalf("SQL = %s", built.Text)
}
if len(built.Args) != 0 || !strings.Contains(built.Text, "protocol = 'GB32960'") || !strings.Contains(built.Text, "vin = 'VIN001'") || !strings.Contains(built.Text, "LIMIT 1 OFFSET 0") {
if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") {
t.Fatalf("count SQL = %s", built.CountText)
}
if len(built.Args) != 0 || !strings.Contains(built.Text, "protocol = 'GB32960'") || !strings.Contains(built.Text, "vin = 'VIN001'") || !strings.Contains(built.Text, "ORDER BY ts DESC, frame_id ASC LIMIT 1 OFFSET 0") {
t.Fatalf("args = %#v", built.Args)
}
}
func TestBuildHistoryLocationSQL(t *testing.T) {
built := buildHistoryLocationSQL("lingniu_vehicle_ts", map[string]string{
"protocol": "JT808",
"vin": "VIN001",
"limit": "2",
"offset": "4",
})
if !strings.Contains(built.Text, "lingniu_vehicle_ts.loc_jt808_") || !strings.Contains(built.Text, "vin = 'VIN001'") {
t.Fatalf("SQL = %s", built.Text)
}
if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") {
t.Fatalf("count SQL = %s", built.CountText)
}
if !strings.Contains(built.Text, "ORDER BY ts DESC, vin ASC, protocol ASC LIMIT 2 OFFSET 4") {
t.Fatalf("SQL should keep stable pagination order: %s", built.Text)
}
}

16
vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go
View File

@@ -29,8 +29,12 @@ func buildRawFrameSQL(database string, query RawFrameQuery) SQLQuery {
if len(where) > 0 {
text += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args}
countText := `SELECT COUNT(*) FROM ` + table
if len(where) > 0 {
countText += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC, frame_id ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args, CountText: countText}
}
func rawFrameWhere(query RawFrameQuery) []string {
@@ -71,8 +75,12 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery
if len(where) > 0 {
text += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args}
countText := `SELECT COUNT(*) FROM ` + table
if len(where) > 0 {
countText += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC, vin ASC, protocol ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args, CountText: countText}
}
func qualifyTDengine(database, table string) string {