shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(platform): paginate history queries

Browse Source
This commit is contained in:
lingniu
2026-07-03 22:19:14 +08:00
parent ea96b6909a
commit 72aa20b347
4 changed files with 91 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
vehicle-data-platform/apps/api/internal/platform/production_store.go
View File

@@ -183,6 +183,12 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer
tdQuery["dateTo"] = value
}
built := buildHistoryLocationSQL(s.tdDatabase, tdQuery)
total := 0
if built.CountText != "" {
if err := s.tdengine.QueryRowContext(ctx, built.CountText, built.CountArgs...).Scan(&total); err != nil {
return Page[HistoryLocationRow]{}, err
}
}
rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...)
if err != nil {
return Page[HistoryLocationRow]{}, err
@@ -207,7 +213,10 @@ func (s *ProductionStore) HistoryLocationsFromTDengine(ctx context.Context, quer
if err := rows.Err(); err != nil {
return Page[HistoryLocationRow]{}, err
}
return Page[HistoryLocationRow]{Items: items, Total: len(items), Limit: limit, Offset: offset}, nil
if built.CountText == "" {
total = len(items)
}
return Page[HistoryLocationRow]{Items: items, Total: total, Limit: limit, Offset: offset}, nil
}
func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (Page[RawFrameRow], error) {
@@ -215,6 +224,12 @@ func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (P
return Page[RawFrameRow]{Items: []RawFrameRow{}, Total: 0, Limit: query.Limit, Offset: query.Offset}, nil
}
built := buildRawFrameSQL(s.tdDatabase, query)
total := 0
if built.CountText != "" {
if err := s.tdengine.QueryRowContext(ctx, built.CountText, built.CountArgs...).Scan(&total); err != nil {
return Page[RawFrameRow]{}, err
}
}
rows, err := s.tdengine.QueryContext(ctx, built.Text, built.Args...)
if err != nil {
return Page[RawFrameRow]{}, err
@@ -244,7 +259,10 @@ func (s *ProductionStore) RawFrames(ctx context.Context, query RawFrameQuery) (P
if err := rows.Err(); err != nil {
return Page[RawFrameRow]{}, err
}
return Page[RawFrameRow]{Items: items, Total: len(items), Limit: query.Limit, Offset: query.Offset}, nil
if built.CountText == "" {
total = len(items)
}
return Page[RawFrameRow]{Items: items, Total: total, Limit: query.Limit, Offset: query.Offset}, nil
}
func (s *ProductionStore) DailyMileage(ctx context.Context, query url.Values) (Page[DailyMileageRow], error) {

23
vehicle-data-platform/apps/api/internal/platform/query_builders_test.go
View File

@@ -80,7 +80,28 @@ func TestBuildRawFrameSQL(t *testing.T) {
if !strings.Contains(built.Text, "lingniu_vehicle_ts.raw_gb32960_") || !strings.Contains(built.Text, "parsed_fields") {
t.Fatalf("SQL = %s", built.Text)
}
if len(built.Args) != 0 || !strings.Contains(built.Text, "protocol = 'GB32960'") || !strings.Contains(built.Text, "vin = 'VIN001'") || !strings.Contains(built.Text, "LIMIT 1 OFFSET 0") {
if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") {
t.Fatalf("count SQL = %s", built.CountText)
}
if len(built.Args) != 0 || !strings.Contains(built.Text, "protocol = 'GB32960'") || !strings.Contains(built.Text, "vin = 'VIN001'") || !strings.Contains(built.Text, "ORDER BY ts DESC, frame_id ASC LIMIT 1 OFFSET 0") {
t.Fatalf("args = %#v", built.Args)
}
}
func TestBuildHistoryLocationSQL(t *testing.T) {
built := buildHistoryLocationSQL("lingniu_vehicle_ts", map[string]string{
"protocol": "JT808",
"vin": "VIN001",
"limit": "2",
"offset": "4",
})
if !strings.Contains(built.Text, "lingniu_vehicle_ts.loc_jt808_") || !strings.Contains(built.Text, "vin = 'VIN001'") {
t.Fatalf("SQL = %s", built.Text)
}
if !strings.Contains(built.CountText, "COUNT(*)") || strings.Contains(built.CountText, "LIMIT") {
t.Fatalf("count SQL = %s", built.CountText)
}
if !strings.Contains(built.Text, "ORDER BY ts DESC, vin ASC, protocol ASC LIMIT 2 OFFSET 4") {
t.Fatalf("SQL should keep stable pagination order: %s", built.Text)
}
}

16
vehicle-data-platform/apps/api/internal/platform/tdengine_queries.go
View File

@@ -29,8 +29,12 @@ func buildRawFrameSQL(database string, query RawFrameQuery) SQLQuery {
if len(where) > 0 {
text += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args}
countText := `SELECT COUNT(*) FROM ` + table
if len(where) > 0 {
countText += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC, frame_id ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args, CountText: countText}
}
func rawFrameWhere(query RawFrameQuery) []string {
@@ -71,8 +75,12 @@ func buildHistoryLocationSQL(database string, query map[string]string) SQLQuery
if len(where) > 0 {
text += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args}
countText := `SELECT COUNT(*) FROM ` + table
if len(where) > 0 {
countText += ` WHERE ` + strings.Join(where, " AND ")
}
text += ` ORDER BY ts DESC, vin ASC, protocol ASC LIMIT ` + strconv.Itoa(limit) + ` OFFSET ` + strconv.Itoa(offset)
return SQLQuery{Text: text, Args: args, CountText: countText}
}
func qualifyTDengine(database, table string) string {

92
vehicle-data-platform/apps/web/src/pages/History.tsx
View File

@@ -1,5 +1,5 @@
import { Button, Card, Form, Select, SideSheet, Space, Table, Tabs, Toast, Typography } from '@douyinfe/semi-ui';
import { IconChevronLeft, IconChevronRight, IconRefresh, IconSearch } from '@douyinfe/semi-icons';
import { IconRefresh, IconSearch } from '@douyinfe/semi-icons';
import { useEffect, useMemo, useState } from 'react';
import { api } from '../api/client';
import type { HistoryLocationRow, Page, RawFrameRow } from '../api/types';
@@ -29,6 +29,8 @@ export function History() {
const [selectedRaw, setSelectedRaw] = useState<RawFrameRow | null>(null);
const [loadingLocations, setLoadingLocations] = useState(false);
const [loadingRaw, setLoadingRaw] = useState(false);
const [locationPagination, setLocationPagination] = useState({ currentPage: 1, pageSize: 10 });
const [rawPagination, setRawPagination] = useState({ currentPage: 1, pageSize: 10 });
const buildParams = (nextFilters: HistoryFilters, limit: number, offset: number, raw: boolean) => {
const params = new URLSearchParams({ limit: String(limit), offset: String(offset) });
@@ -41,18 +43,24 @@ export function History() {
return params;
};
const loadLocations = (nextFilters = filters, offset = locations.offset) => {
const loadLocations = (nextFilters = filters, page = locationPagination.currentPage, pageSize = locationPagination.pageSize) => {
setLoadingLocations(true);
api.historyLocations(buildParams(nextFilters, locations.limit || 10, offset, false))
.then(setLocations)
api.historyLocations(buildParams(nextFilters, pageSize, (page - 1) * pageSize, false))
.then((nextPage) => {
setLocations(nextPage);
setLocationPagination({ currentPage: page, pageSize });
})
.catch((error: Error) => Toast.error(error.message))
.finally(() => setLoadingLocations(false));
};
const loadRawFrames = (nextFilters = filters, offset = rawFrames.offset) => {
const loadRawFrames = (nextFilters = filters, page = rawPagination.currentPage, pageSize = rawPagination.pageSize) => {
setLoadingRaw(true);
api.rawFrames(buildParams(nextFilters, rawFrames.limit || 10, offset, true))
.then(setRawFrames)
api.rawFrames(buildParams(nextFilters, pageSize, (page - 1) * pageSize, true))
.then((nextPage) => {
setRawFrames(nextPage);
setRawPagination({ currentPage: page, pageSize });
})
.catch((error: Error) => Toast.error(error.message))
.finally(() => setLoadingRaw(false));
};
@@ -67,19 +75,19 @@ export function History() {
includeFields: Boolean(values.includeFields)
};
setFilters(nextFilters);
loadLocations(nextFilters, 0);
loadRawFrames(nextFilters, 0);
loadLocations(nextFilters, 1, locationPagination.pageSize);
loadRawFrames(nextFilters, 1, rawPagination.pageSize);
};
const reset = () => {
setFilters(defaultFilters);
loadLocations(defaultFilters, 0);
loadRawFrames(defaultFilters, 0);
loadLocations(defaultFilters, 1, locationPagination.pageSize);
loadRawFrames(defaultFilters, 1, rawPagination.pageSize);
};
useEffect(() => {
loadLocations(defaultFilters, 0);
loadRawFrames(defaultFilters, 0);
loadLocations(defaultFilters, 1, locationPagination.pageSize);
loadRawFrames(defaultFilters, 1, rawPagination.pageSize);
}, []);
const rawFieldCount = useMemo(() => Object.keys(selectedRaw?.parsedFields ?? {}).length, [selectedRaw]);
@@ -120,7 +128,14 @@ export function History() {
rowKey="deviceTime"
dataSource={locations.items}
loading={loadingLocations}
pagination={false}
pagination={{
currentPage: locationPagination.currentPage,
pageSize: locationPagination.pageSize,
total: locations.total,
showSizeChanger: true,
onPageChange: (page) => loadLocations(filters, page, locationPagination.pageSize),
onPageSizeChange: (pageSize) => loadLocations(filters, 1, pageSize)
}}
columns={[
{ title: 'VIN', dataIndex: 'vin', width: 190 },
{ title: '协议', dataIndex: 'protocol', width: 120 },
@@ -132,19 +147,20 @@ export function History() {
{ title: '入库时间', dataIndex: 'serverTime', width: 190 }
]}
/>
<HistoryPager
page={locations}
loading={loadingLocations}
onPrev={() => loadLocations(filters, Math.max(0, locations.offset - locations.limit))}
onNext={() => loadLocations(filters, locations.offset + locations.limit)}
/>
</Tabs.TabPane>
<Tabs.TabPane tab="RAW 帧" itemKey="raw">
<Table
rowKey="id"
dataSource={rawFrames.items}
loading={loadingRaw}
pagination={false}
pagination={{
currentPage: rawPagination.currentPage,
pageSize: rawPagination.pageSize,
total: rawFrames.total,
showSizeChanger: true,
onPageChange: (page) => loadRawFrames(filters, page, rawPagination.pageSize),
onPageSizeChange: (pageSize) => loadRawFrames(filters, 1, pageSize)
}}
columns={[
{ title: 'ID', dataIndex: 'id', width: 260 },
{ title: 'VIN', dataIndex: 'vin', width: 190 },
@@ -156,12 +172,6 @@ export function History() {
{ title: '操作', width: 100, render: (_: unknown, row: RawFrameRow) => <Button onClick={() => setSelectedRaw(row)}>字段</Button> }
]}
/>
<HistoryPager
page={rawFrames}
loading={loadingRaw}
onPrev={() => loadRawFrames(filters, Math.max(0, rawFrames.offset - rawFrames.limit))}
onNext={() => loadRawFrames(filters, rawFrames.offset + rawFrames.limit)}
/>
</Tabs.TabPane>
</Tabs>
</Card>
@@ -179,31 +189,3 @@ export function History() {
</div>
);
}
function HistoryPager<T>({
page,
loading,
onPrev,
onNext
}: {
page: Page<T>;
loading: boolean;
onPrev: () => void;
onNext: () => void;
}) {
const start = page.offset + 1;
const end = page.offset + page.items.length;
const hasPrevious = page.offset > 0;
const hasNext = page.items.length >= page.limit;
return (
<div className="vp-table-footer">
<Typography.Text type="tertiary">
当前 {page.items.length === 0 ? 0 : start}-{end} 条，每页 {page.limit} 条
</Typography.Text>
<Space>
<Button icon={<IconChevronLeft />} disabled={!hasPrevious || loading} onClick={onPrev}>上一页</Button>
<Button icon={<IconChevronRight />} disabled={!hasNext || loading} onClick={onNext}>下一页</Button>
</Space>
</div>
);
}