shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: 优化监控刷新与历史里程分页查询

Browse Source
This commit is contained in:
lingniu
2026-09-09 10:13:44 +08:00
parent 1849bc1870
commit 9f0a87f49e
7 changed files with 171 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vehicle-data-platform/apps/api/internal/platform/daily_mileage_pagination_test.go
+48
View File
@@ -0,0 +1,48 @@
package platform
import (
"net/url"
"reflect"
"strings"
"testing"
)
func TestDailyMileageEnrichesOnlyTheSelectedPage(t *testing.T) {
for _, dedup := range []string{"0", "1"} {
q := url.Values{"deduplicate": {dedup}, "limit": {"7"}, "offset": {"21"}, "scopeVins": {"VIN1,VIN2"}, "dateFrom": {"2026-08-01"}, "dateTo": {"2026-08-31"}, "protocols": {"JT808,GB32960"}}
got := buildDailyMileageSQL(q)
pageEnd := strings.Index(got.Text, "LIMIT ? OFFSET ?)")
energy := strings.Index(got.Text, "LEFT JOIN vehicle_open_daily_energy")
binding := strings.Index(got.Text, "LEFT JOIN vehicle_identity_binding")
if pageEnd < 0 || energy < pageEnd || binding < pageEnd {
t.Fatalf("enrichment precedes page: %s", got.Text)
}
if strings.Contains(got.CountText, "JOIN") || strings.Contains(got.CountText, "LIMIT") {
t.Fatalf("unneeded joins or pagination in total: %s", got.CountText)
}
if !reflect.DeepEqual(got.CountArgs, got.Args[:len(got.Args)-2]) || got.Args[len(got.Args)-2] != 7 || got.Args[len(got.Args)-1] != 21 {
t.Fatalf("args data=%#v count=%#v", got.Args, got.CountArgs)
}
for _, predicate := range []string{"m.vin IN (?,?)", "m.protocol IN (?,?)", "m.stat_date >= ?", "m.stat_date <= ?"} {
if !strings.Contains(got.Text, predicate) || !strings.Contains(got.CountText, predicate) {
t.Fatalf("lost %s", predicate)
}
}
}
}
func TestDailyMileageCountKeepsBindingOnlyForBindingPredicates(t *testing.T) {
for _, q := range []url.Values{{"vin": {"fleet"}}, {"vehicleScope": {"bound"}}, {"vin": {"fleet"}, "deduplicate": {"1"}}, {"vehicleScope": {"bound"}, "deduplicate": {"1"}}} {
got := buildDailyMileageSQL(q)
if !strings.Contains(got.CountText, "vehicle_identity_binding b") || strings.Contains(got.CountText, "vehicle_open_daily_energy") {
t.Fatal(got.CountText)
}
if strings.Index(got.Text, "vehicle_identity_binding b") > strings.Index(got.Text, "LIMIT ? OFFSET ?)") {
t.Fatal("binding filter applied after pagination")
}
}
got := buildDailyMileageSQL(url.Values{})
if len(got.CountArgs) != 0 || strings.Contains(got.CountText, "stat_date >=") || strings.Contains(got.CountText, "stat_date <=") {
t.Fatal("unfiltered history total changed")
}
}
vehicle-data-platform/apps/api/internal/platform/monitor_activity.go
+18
View File
@@ -0,0 +1,18 @@
package platform
import (
"context"
"time"
)
// MonitorActivitySummary reads only the two global counters used by the monitor.
// A viewport refresh must not run the operations dashboard's health probes.
func (s *ProductionStore) MonitorActivitySummary(ctx context.Context) (DashboardSummary, error) {
var result DashboardSummary
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT vin) FROM vehicle_realtime_snapshot WHERE vin IS NOT NULL AND vin <> '' AND updated_at >= CURDATE()`).Scan(&result.ActiveToday); err != nil {
return result, err
}
frames, err := s.frameToday(ctx, time.Now())
result.FrameToday = frames
return result, err
}
vehicle-data-platform/apps/api/internal/platform/monitor_activity_test.go
+33
View File
@@ -0,0 +1,33 @@
package platform
import (
"context"
"net/url"
"testing"
)
type monitorActivityTestStore struct {
*MockStore
activityCalls int
}
func (s *monitorActivityTestStore) MonitorActivitySummary(context.Context) (DashboardSummary, error) {
s.activityCalls++
return DashboardSummary{ActiveToday: 17, FrameToday: 123456}, nil
}
func (s *monitorActivityTestStore) DashboardSummary(context.Context) (DashboardSummary, error) {
panic("monitor must not run full dashboard and health probes")
}
func TestMonitorWorkspaceUsesOnlyRequiredActivityCounters(t *testing.T) {
store := &monitorActivityTestStore{MockStore: NewMockStore()}
got, err := NewService(store).MonitorWorkspace(context.Background(), url.Values{"limit": {"10000"}, "zoom": {"5"}, "railLimit": {"200"}})
if err != nil {
t.Fatal(err)
}
if store.activityCalls != 1 || got.Summary.ActiveToday != 17 || got.Summary.FrameToday != 123456 {
t.Fatalf("calls=%d summary=%+v", store.activityCalls, got.Summary)
}
if got.Summary.TotalVehicles == 0 || len(got.Vehicles.Items) == 0 {
t.Fatal("lost fleet data")
}
}
vehicle-data-platform/apps/api/internal/platform/mysql_queries.go
+31 -29
View File
@@ -554,13 +554,19 @@ func buildDailyMileageSQL(query url.Values) SQLQuery {
}
countArgs := append([]any(nil), args...)
args = append(args, limit, offset)
fromSQL := `FROM vehicle_daily_mileage m
LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin
// Both joins are one-to-one. Binding is needed during filtering only for
// keyword/bound scope; energy never affects which mileage rows qualify.
filterFromSQL := `FROM vehicle_daily_mileage m`
if strings.EqualFold(strings.TrimSpace(query.Get("vehicleScope")), "bound") || strings.TrimSpace(query.Get("vin")) != "" {
filterFromSQL += ` LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin`
}
filterFromSQL += ` WHERE ` + strings.Join(where, " AND ")
enrichmentSQL := ` LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin
LEFT JOIN vehicle_open_daily_energy h
ON h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci
AND h.stat_date = m.stat_date
AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT')
WHERE ` + strings.Join(where, " AND ")
AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT')`
if query.Get("deduplicate") == "1" || strings.EqualFold(query.Get("deduplicate"), "true") {
selectionOrder := `m.daily_mileage_km DESC, m.protocol ASC`
dailyMileageExpression := `MAX(COALESCE(m.daily_mileage_km, 0))`
@@ -570,30 +576,26 @@ WHERE ` + strings.Join(where, " AND ")
dailyMileageExpression = `COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(COALESCE(m.daily_mileage_km, 0) AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0)`
pureHydrogenMileageExpression = `COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(COALESCE(m.pure_hydrogen_mileage_km, 0) AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0)`
}
groupSQL := fromSQL + ` GROUP BY m.vin, m.stat_date`
groupSQL := filterFromSQL + ` GROUP BY m.vin, m.stat_date`
// LIMIT materializes the grouped page before enrichment. Keep the exact
// original aggregate/priority rules, including independent maxima when no
// enabled protocol priority was supplied.
pageSQL := `SELECT m.vin, m.stat_date, ` +
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km - m.daily_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS start_mileage_km, ` +
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS end_mileage_km, ` +
dailyMileageExpression + ` AS daily_mileage_km, ` + pureHydrogenMileageExpression + ` AS pure_hydrogen_mileage_km, ` +
`COALESCE(SUBSTRING_INDEX(GROUP_CONCAT(m.protocol ORDER BY ` + selectionOrder + `), ',', 1), '') AS protocol ` +
groupSQL + ` ORDER BY m.stat_date DESC, m.vin ASC LIMIT ? OFFSET ?`
built := SQLQuery{
Text: `SELECT m.vin, COALESCE(MAX(NULLIF(b.plate, '')), '') AS plate, DATE_FORMAT(m.stat_date, '%Y-%m-%d') AS stat_date, ` +
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km - m.daily_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS start_mileage_km, ` +
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS end_mileage_km, ` +
dailyMileageExpression + ` AS daily_mileage_km, ` +
pureHydrogenMileageExpression + ` AS pure_hydrogen_mileage_km, ` +
`MAX(h.consumption_kg) AS hydrogen_consumption_kg, ` +
`MAX(h.consumption_kg_per_100km) AS hydrogen_consumption_kg_per_100km, ` +
`MAX(h.soc_balanced_consumption_kg) AS hydrogen_soc_balanced_kg, ` +
`MAX(h.soc_balanced_kg_per_100km) AS hydrogen_soc_balanced_kg_per_100km, ` +
`MAX(h.pure_electric_mileage_km) AS pure_electric_mileage_km, ` +
`MAX(h.mixed_mileage_km) AS mixed_mileage_km, ` +
`MAX(h.battery_soc_delta_pct) AS battery_soc_delta_pct, ` +
`MAX(h.charge_count) AS charge_count, ` +
`MAX(h.charge_energy_kwh) AS charge_energy_kwh, ` +
`MAX(h.refuel_count) AS refuel_count, ` +
`MAX(h.refuel_amount_kg) AS refuel_amount_kg, ` +
`MAX(CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END) AS hydrogen_evidence_available, ` +
`COALESCE(MAX(h.quality_status), '') AS hydrogen_quality_status, ` +
`COALESCE(MAX(h.quality_reason), '') AS hydrogen_quality_reason, ` +
`COALESCE(MAX(h.algorithm_version), '') AS hydrogen_algorithm_version, ` +
`COALESCE(SUBSTRING_INDEX(GROUP_CONCAT(m.protocol ORDER BY ` + selectionOrder + `), ',', 1), '') AS protocol ` +
groupSQL + ` ORDER BY m.stat_date DESC, m.vin ASC LIMIT ? OFFSET ?`,
Text: `SELECT m.vin, COALESCE(NULLIF(b.plate, ''), '') AS plate, DATE_FORMAT(m.stat_date, '%Y-%m-%d') AS stat_date, ` +
`m.start_mileage_km, m.end_mileage_km, m.daily_mileage_km, m.pure_hydrogen_mileage_km, ` +
`h.consumption_kg AS hydrogen_consumption_kg, h.consumption_kg_per_100km AS hydrogen_consumption_kg_per_100km, ` +
`h.soc_balanced_consumption_kg AS hydrogen_soc_balanced_kg, h.soc_balanced_kg_per_100km AS hydrogen_soc_balanced_kg_per_100km, ` +
`h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, ` +
`CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END AS hydrogen_evidence_available, ` +
`COALESCE(h.quality_status, '') AS hydrogen_quality_status, COALESCE(h.quality_reason, '') AS hydrogen_quality_reason, ` +
`COALESCE(h.algorithm_version, '') AS hydrogen_algorithm_version, m.protocol ` +
`FROM (` + pageSQL + `) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC`,
Args: args,
CountText: `SELECT COUNT(*) FROM (SELECT m.vin ` + groupSQL + `) vehicle_daily_mileage_count`,
CountArgs: countArgs,
@@ -612,9 +614,9 @@ WHERE ` + strings.Join(where, " AND ")
`h.soc_balanced_kg_per_100km, h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, ` +
`h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END, ` +
`COALESCE(h.quality_status, ''), COALESCE(h.quality_reason, ''), COALESCE(h.algorithm_version, ''), m.protocol ` +
fromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?`,
`FROM (SELECT m.* ` + filterFromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC`,
Args: args,
CountText: `SELECT COUNT(*) ` + fromSQL,
CountText: `SELECT COUNT(*) ` + filterFromSQL,
CountArgs: countArgs,
}
if query.Get("skipCount") == "1" || strings.EqualFold(query.Get("skipCount"), "true") {
vehicle-data-platform/apps/api/internal/platform/query_builders_test.go
+1 -1
View File
@@ -568,7 +568,7 @@ func TestMileageQueriesCanRestrictFleetScopeToAuthoritativelyBoundVehicles(t *te
func TestBuildDailyMileageSQLCanMatchStatisticsVehicleDayScope(t *testing.T) {
built := buildDailyMileageSQL(url.Values{"deduplicate": {"1"}, "limit": {"50"}})
for _, want := range []string{"GROUP BY m.vin, m.stat_date", "MAX(COALESCE(m.daily_mileage_km, 0))", "MAX(COALESCE(m.pure_hydrogen_mileage_km, 0))", "MAX(h.consumption_kg)", "h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci", "GROUP_CONCAT(m.protocol ORDER BY m.daily_mileage_km DESC", "vehicle_daily_mileage_count"} {
for _, want := range []string{"GROUP BY m.vin, m.stat_date", "MAX(COALESCE(m.daily_mileage_km, 0))", "MAX(COALESCE(m.pure_hydrogen_mileage_km, 0))", "h.consumption_kg AS hydrogen_consumption_kg", "h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci", "GROUP_CONCAT(m.protocol ORDER BY m.daily_mileage_km DESC", "vehicle_daily_mileage_count"} {
if !strings.Contains(built.Text+built.CountText, want) {
t.Fatalf("deduplicated daily mileage SQL missing %q: %s / %s", want, built.Text, built.CountText)
}
vehicle-data-platform/apps/api/internal/platform/service.go
+11 -1
View File
@@ -544,8 +544,18 @@ func (s *Service) MonitorSummary(ctx context.Context, query url.Values) (Monitor
return s.buildMonitorSummary(ctx, query, vehicles)
}
type monitorActivityStore interface {
MonitorActivitySummary(context.Context) (DashboardSummary, error)
}
func (s *Service) buildMonitorSummary(ctx context.Context, query url.Values, vehicles Page[VehicleRealtimeRow]) (MonitorSummary, error) {
dashboard, err := s.store.DashboardSummary(ctx)
var dashboard DashboardSummary
var err error
if store, ok := s.store.(monitorActivityStore); ok {
dashboard, err = store.MonitorActivitySummary(ctx)
} else {
dashboard, err = s.store.DashboardSummary(ctx)
}
if err != nil {
return MonitorSummary{}, err
}