perf: 优化监控刷新与历史里程分页查询
This commit is contained in:
lingniu
@@ -0,0 +1,48 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDailyMileageEnrichesOnlyTheSelectedPage(t *testing.T) {
|
||||
for _, dedup := range []string{"0", "1"} {
|
||||
q := url.Values{"deduplicate": {dedup}, "limit": {"7"}, "offset": {"21"}, "scopeVins": {"VIN1,VIN2"}, "dateFrom": {"2026-08-01"}, "dateTo": {"2026-08-31"}, "protocols": {"JT808,GB32960"}}
|
||||
got := buildDailyMileageSQL(q)
|
||||
pageEnd := strings.Index(got.Text, "LIMIT ? OFFSET ?)")
|
||||
energy := strings.Index(got.Text, "LEFT JOIN vehicle_open_daily_energy")
|
||||
binding := strings.Index(got.Text, "LEFT JOIN vehicle_identity_binding")
|
||||
if pageEnd < 0 || energy < pageEnd || binding < pageEnd {
|
||||
t.Fatalf("enrichment precedes page: %s", got.Text)
|
||||
}
|
||||
if strings.Contains(got.CountText, "JOIN") || strings.Contains(got.CountText, "LIMIT") {
|
||||
t.Fatalf("unneeded joins or pagination in total: %s", got.CountText)
|
||||
}
|
||||
if !reflect.DeepEqual(got.CountArgs, got.Args[:len(got.Args)-2]) || got.Args[len(got.Args)-2] != 7 || got.Args[len(got.Args)-1] != 21 {
|
||||
t.Fatalf("args data=%#v count=%#v", got.Args, got.CountArgs)
|
||||
}
|
||||
for _, predicate := range []string{"m.vin IN (?,?)", "m.protocol IN (?,?)", "m.stat_date >= ?", "m.stat_date <= ?"} {
|
||||
if !strings.Contains(got.Text, predicate) || !strings.Contains(got.CountText, predicate) {
|
||||
t.Fatalf("lost %s", predicate)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailyMileageCountKeepsBindingOnlyForBindingPredicates(t *testing.T) {
|
||||
for _, q := range []url.Values{{"vin": {"fleet"}}, {"vehicleScope": {"bound"}}, {"vin": {"fleet"}, "deduplicate": {"1"}}, {"vehicleScope": {"bound"}, "deduplicate": {"1"}}} {
|
||||
got := buildDailyMileageSQL(q)
|
||||
if !strings.Contains(got.CountText, "vehicle_identity_binding b") || strings.Contains(got.CountText, "vehicle_open_daily_energy") {
|
||||
t.Fatal(got.CountText)
|
||||
}
|
||||
if strings.Index(got.Text, "vehicle_identity_binding b") > strings.Index(got.Text, "LIMIT ? OFFSET ?)") {
|
||||
t.Fatal("binding filter applied after pagination")
|
||||
}
|
||||
}
|
||||
got := buildDailyMileageSQL(url.Values{})
|
||||
if len(got.CountArgs) != 0 || strings.Contains(got.CountText, "stat_date >=") || strings.Contains(got.CountText, "stat_date <=") {
|
||||
t.Fatal("unfiltered history total changed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MonitorActivitySummary reads only the two global counters used by the monitor.
|
||||
// A viewport refresh must not run the operations dashboard's health probes.
|
||||
func (s *ProductionStore) MonitorActivitySummary(ctx context.Context) (DashboardSummary, error) {
|
||||
var result DashboardSummary
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(DISTINCT vin) FROM vehicle_realtime_snapshot WHERE vin IS NOT NULL AND vin <> '' AND updated_at >= CURDATE()`).Scan(&result.ActiveToday); err != nil {
|
||||
return result, err
|
||||
}
|
||||
frames, err := s.frameToday(ctx, time.Now())
|
||||
result.FrameToday = frames
|
||||
return result, err
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package platform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type monitorActivityTestStore struct {
|
||||
*MockStore
|
||||
activityCalls int
|
||||
}
|
||||
|
||||
func (s *monitorActivityTestStore) MonitorActivitySummary(context.Context) (DashboardSummary, error) {
|
||||
s.activityCalls++
|
||||
return DashboardSummary{ActiveToday: 17, FrameToday: 123456}, nil
|
||||
}
|
||||
func (s *monitorActivityTestStore) DashboardSummary(context.Context) (DashboardSummary, error) {
|
||||
panic("monitor must not run full dashboard and health probes")
|
||||
}
|
||||
func TestMonitorWorkspaceUsesOnlyRequiredActivityCounters(t *testing.T) {
|
||||
store := &monitorActivityTestStore{MockStore: NewMockStore()}
|
||||
got, err := NewService(store).MonitorWorkspace(context.Background(), url.Values{"limit": {"10000"}, "zoom": {"5"}, "railLimit": {"200"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if store.activityCalls != 1 || got.Summary.ActiveToday != 17 || got.Summary.FrameToday != 123456 {
|
||||
t.Fatalf("calls=%d summary=%+v", store.activityCalls, got.Summary)
|
||||
}
|
||||
if got.Summary.TotalVehicles == 0 || len(got.Vehicles.Items) == 0 {
|
||||
t.Fatal("lost fleet data")
|
||||
}
|
||||
}
|
||||
@@ -554,13 +554,19 @@ func buildDailyMileageSQL(query url.Values) SQLQuery {
|
||||
}
|
||||
countArgs := append([]any(nil), args...)
|
||||
args = append(args, limit, offset)
|
||||
fromSQL := `FROM vehicle_daily_mileage m
|
||||
LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin
|
||||
// Both joins are one-to-one. Binding is needed during filtering only for
|
||||
// keyword/bound scope; energy never affects which mileage rows qualify.
|
||||
filterFromSQL := `FROM vehicle_daily_mileage m`
|
||||
if strings.EqualFold(strings.TrimSpace(query.Get("vehicleScope")), "bound") || strings.TrimSpace(query.Get("vin")) != "" {
|
||||
filterFromSQL += ` LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin`
|
||||
}
|
||||
filterFromSQL += ` WHERE ` + strings.Join(where, " AND ")
|
||||
enrichmentSQL := ` LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin
|
||||
LEFT JOIN vehicle_open_daily_energy h
|
||||
ON h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci
|
||||
AND h.stat_date = m.stat_date
|
||||
AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT')
|
||||
WHERE ` + strings.Join(where, " AND ")
|
||||
AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT')`
|
||||
|
||||
if query.Get("deduplicate") == "1" || strings.EqualFold(query.Get("deduplicate"), "true") {
|
||||
selectionOrder := `m.daily_mileage_km DESC, m.protocol ASC`
|
||||
dailyMileageExpression := `MAX(COALESCE(m.daily_mileage_km, 0))`
|
||||
@@ -570,30 +576,26 @@ WHERE ` + strings.Join(where, " AND ")
|
||||
dailyMileageExpression = `COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(COALESCE(m.daily_mileage_km, 0) AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0)`
|
||||
pureHydrogenMileageExpression = `COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(COALESCE(m.pure_hydrogen_mileage_km, 0) AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0)`
|
||||
}
|
||||
groupSQL := fromSQL + ` GROUP BY m.vin, m.stat_date`
|
||||
groupSQL := filterFromSQL + ` GROUP BY m.vin, m.stat_date`
|
||||
// LIMIT materializes the grouped page before enrichment. Keep the exact
|
||||
// original aggregate/priority rules, including independent maxima when no
|
||||
// enabled protocol priority was supplied.
|
||||
pageSQL := `SELECT m.vin, m.stat_date, ` +
|
||||
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km - m.daily_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS start_mileage_km, ` +
|
||||
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS end_mileage_km, ` +
|
||||
dailyMileageExpression + ` AS daily_mileage_km, ` + pureHydrogenMileageExpression + ` AS pure_hydrogen_mileage_km, ` +
|
||||
`COALESCE(SUBSTRING_INDEX(GROUP_CONCAT(m.protocol ORDER BY ` + selectionOrder + `), ',', 1), '') AS protocol ` +
|
||||
groupSQL + ` ORDER BY m.stat_date DESC, m.vin ASC LIMIT ? OFFSET ?`
|
||||
built := SQLQuery{
|
||||
Text: `SELECT m.vin, COALESCE(MAX(NULLIF(b.plate, '')), '') AS plate, DATE_FORMAT(m.stat_date, '%Y-%m-%d') AS stat_date, ` +
|
||||
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km - m.daily_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS start_mileage_km, ` +
|
||||
`COALESCE(CAST(SUBSTRING_INDEX(GROUP_CONCAT(CAST(m.latest_total_mileage_km AS CHAR) ORDER BY ` + selectionOrder + `), ',', 1) AS DECIMAL(18,3)), 0) AS end_mileage_km, ` +
|
||||
dailyMileageExpression + ` AS daily_mileage_km, ` +
|
||||
pureHydrogenMileageExpression + ` AS pure_hydrogen_mileage_km, ` +
|
||||
`MAX(h.consumption_kg) AS hydrogen_consumption_kg, ` +
|
||||
`MAX(h.consumption_kg_per_100km) AS hydrogen_consumption_kg_per_100km, ` +
|
||||
`MAX(h.soc_balanced_consumption_kg) AS hydrogen_soc_balanced_kg, ` +
|
||||
`MAX(h.soc_balanced_kg_per_100km) AS hydrogen_soc_balanced_kg_per_100km, ` +
|
||||
`MAX(h.pure_electric_mileage_km) AS pure_electric_mileage_km, ` +
|
||||
`MAX(h.mixed_mileage_km) AS mixed_mileage_km, ` +
|
||||
`MAX(h.battery_soc_delta_pct) AS battery_soc_delta_pct, ` +
|
||||
`MAX(h.charge_count) AS charge_count, ` +
|
||||
`MAX(h.charge_energy_kwh) AS charge_energy_kwh, ` +
|
||||
`MAX(h.refuel_count) AS refuel_count, ` +
|
||||
`MAX(h.refuel_amount_kg) AS refuel_amount_kg, ` +
|
||||
`MAX(CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END) AS hydrogen_evidence_available, ` +
|
||||
`COALESCE(MAX(h.quality_status), '') AS hydrogen_quality_status, ` +
|
||||
`COALESCE(MAX(h.quality_reason), '') AS hydrogen_quality_reason, ` +
|
||||
`COALESCE(MAX(h.algorithm_version), '') AS hydrogen_algorithm_version, ` +
|
||||
`COALESCE(SUBSTRING_INDEX(GROUP_CONCAT(m.protocol ORDER BY ` + selectionOrder + `), ',', 1), '') AS protocol ` +
|
||||
groupSQL + ` ORDER BY m.stat_date DESC, m.vin ASC LIMIT ? OFFSET ?`,
|
||||
Text: `SELECT m.vin, COALESCE(NULLIF(b.plate, ''), '') AS plate, DATE_FORMAT(m.stat_date, '%Y-%m-%d') AS stat_date, ` +
|
||||
`m.start_mileage_km, m.end_mileage_km, m.daily_mileage_km, m.pure_hydrogen_mileage_km, ` +
|
||||
`h.consumption_kg AS hydrogen_consumption_kg, h.consumption_kg_per_100km AS hydrogen_consumption_kg_per_100km, ` +
|
||||
`h.soc_balanced_consumption_kg AS hydrogen_soc_balanced_kg, h.soc_balanced_kg_per_100km AS hydrogen_soc_balanced_kg_per_100km, ` +
|
||||
`h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, ` +
|
||||
`CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END AS hydrogen_evidence_available, ` +
|
||||
`COALESCE(h.quality_status, '') AS hydrogen_quality_status, COALESCE(h.quality_reason, '') AS hydrogen_quality_reason, ` +
|
||||
`COALESCE(h.algorithm_version, '') AS hydrogen_algorithm_version, m.protocol ` +
|
||||
`FROM (` + pageSQL + `) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC`,
|
||||
Args: args,
|
||||
CountText: `SELECT COUNT(*) FROM (SELECT m.vin ` + groupSQL + `) vehicle_daily_mileage_count`,
|
||||
CountArgs: countArgs,
|
||||
@@ -612,9 +614,9 @@ WHERE ` + strings.Join(where, " AND ")
|
||||
`h.soc_balanced_kg_per_100km, h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, ` +
|
||||
`h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END, ` +
|
||||
`COALESCE(h.quality_status, ''), COALESCE(h.quality_reason, ''), COALESCE(h.algorithm_version, ''), m.protocol ` +
|
||||
fromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?`,
|
||||
`FROM (SELECT m.* ` + filterFromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC`,
|
||||
Args: args,
|
||||
CountText: `SELECT COUNT(*) ` + fromSQL,
|
||||
CountText: `SELECT COUNT(*) ` + filterFromSQL,
|
||||
CountArgs: countArgs,
|
||||
}
|
||||
if query.Get("skipCount") == "1" || strings.EqualFold(query.Get("skipCount"), "true") {
|
||||
|
||||
@@ -568,7 +568,7 @@ func TestMileageQueriesCanRestrictFleetScopeToAuthoritativelyBoundVehicles(t *te
|
||||
|
||||
func TestBuildDailyMileageSQLCanMatchStatisticsVehicleDayScope(t *testing.T) {
|
||||
built := buildDailyMileageSQL(url.Values{"deduplicate": {"1"}, "limit": {"50"}})
|
||||
for _, want := range []string{"GROUP BY m.vin, m.stat_date", "MAX(COALESCE(m.daily_mileage_km, 0))", "MAX(COALESCE(m.pure_hydrogen_mileage_km, 0))", "MAX(h.consumption_kg)", "h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci", "GROUP_CONCAT(m.protocol ORDER BY m.daily_mileage_km DESC", "vehicle_daily_mileage_count"} {
|
||||
for _, want := range []string{"GROUP BY m.vin, m.stat_date", "MAX(COALESCE(m.daily_mileage_km, 0))", "MAX(COALESCE(m.pure_hydrogen_mileage_km, 0))", "h.consumption_kg AS hydrogen_consumption_kg", "h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci", "GROUP_CONCAT(m.protocol ORDER BY m.daily_mileage_km DESC", "vehicle_daily_mileage_count"} {
|
||||
if !strings.Contains(built.Text+built.CountText, want) {
|
||||
t.Fatalf("deduplicated daily mileage SQL missing %q: %s / %s", want, built.Text, built.CountText)
|
||||
}
|
||||
|
||||
@@ -544,8 +544,18 @@ func (s *Service) MonitorSummary(ctx context.Context, query url.Values) (Monitor
|
||||
return s.buildMonitorSummary(ctx, query, vehicles)
|
||||
}
|
||||
|
||||
type monitorActivityStore interface {
|
||||
MonitorActivitySummary(context.Context) (DashboardSummary, error)
|
||||
}
|
||||
|
||||
func (s *Service) buildMonitorSummary(ctx context.Context, query url.Values, vehicles Page[VehicleRealtimeRow]) (MonitorSummary, error) {
|
||||
dashboard, err := s.store.DashboardSummary(ctx)
|
||||
var dashboard DashboardSummary
|
||||
var err error
|
||||
if store, ok := s.store.(monitorActivityStore); ok {
|
||||
dashboard, err = store.MonitorActivitySummary(ctx)
|
||||
} else {
|
||||
dashboard, err = s.store.DashboardSummary(ctx)
|
||||
}
|
||||
if err != nil {
|
||||
return MonitorSummary{}, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# 车辆平台性能修复验收（2026-09-09）
|
||||
|
||||
## 发布
|
||||
|
||||
- 正式版本：`platform-performance-202609091012`，北京时间2026-09-09 10:11:48确认就绪。
|
||||
- 服务：`lingniu-vehicle-platform`；正式域名 https://vehicle.d.lnoneos.com/。
|
||||
- 二进制SHA-256：`3c3358a438e421e89fbe2a39f6b74869ab170a4aee9c04b321e0bc9c9807afe5`，候选与正式一致。
|
||||
- 保留上一版`hydrogen-detail-20260904162007`。原前端资源沿用，发布兼容资源检查通过；无数据库迁移、索引变更或权限调整。
|
||||
|
||||
## 原因与修复
|
||||
|
||||
1. `/api/v2/monitor/workspace`每次地图刷新经由DashboardSummary执行完整运维概览，包括MySQL、TDengine、Redis及容量健康探测和其他用不到的统计。改为读取监控实际需要的当日活跃车数和原始帧数；车辆快照、地图、告警数及列表的原计算保留。监控不再依赖额外健康检查的耗时，但运维健康接口仍保留检查。
|
||||
2. `/api/mileage/daily`在分页之前关联全历史里程与绑定、能源表；计数也执行不影响行数的能源关联。生产EXPLAIN估算154万行，修复后未过滤精确总数为1,617,910。改为先按原条件筛选和分页，再补充能源/车牌字段；仅绑定条件确实参与筛选时保留计数中的绑定关联。
|
||||
3. 保留完整总数、授权VIN/有效日期限制、关键词、协议优先、去重、排序、分页及氢耗字段语义。不通过默认收窄日期或省略记录伪造提速。
|
||||
|
||||
## 测试与测量
|
||||
|
||||
- API模块`go test ./...`、`go test -race ./internal/platform`、`go vet ./internal/platform`、发布脚本自测及diff-check通过。
|
||||
- 在生产机器127.0.0.1:20301隔离候选进程读取真实库，与原20300接口比较四组封闭历史查询的完整data：单日、近一个月第二页、去重、JT808/GB32960协议优先。全部记录、数量、总数和字段逐项一致。
|
||||
- 同一监控参数：`limit=10000&zoom=5&bounds=82.483090,16.144931,128.317906,51.735158&railLimit=200`。原版本本机HTTP三轮631/569/581毫秒；候选102/97/98毫秒。
|
||||
- 正式HTTPS三轮监控126/135/137毫秒；2026-09-01单日里程POST为24/22/22毫秒；2026-08-11至2026-09-08里程POST为163/155/157毫秒（每页20条）。所有请求200。
|
||||
- 未筛选全历史GET：原版本正式接口5秒超时，直接查询超过8秒；正式新版本约1.18秒，完整总数1,617,910。它仍是秒级查询，不能将有日期窗口的验收数字用于全历史承诺。
|
||||
- 鉴权session接口本机3毫秒、HTTPS16毫秒；初步主机负载和内存未见资源耗尽。已复现的慢点位于车辆平台查询和监控运维探测路径。
|
||||
|
||||
## 范围与证据
|
||||
|
||||
用户提供了监控完整URL和里程接口地址，未提供里程POST Payload。因此上述里程测量明确标注了测试日期窗口，尚不能等同于用户原始筛选条件。浏览器只到达登录页，未取得用户会话做界面操作验收；正式接口验证使用运行环境已有管理员诊断凭证，未新建账户或扩大权限，凭证不进入输出。
|
||||
|
||||
代码专项测试覆盖分页先于能源补全、完整计数移除无关关联、必要绑定筛选和参数映射，以及监控只读必要计数、不触发完整健康检查。隔离候选验收结束后停止。脱敏测量、SQL执行计划、比较探针、部署与SHA记录保存在`outputs/platform-perf-20260909/`。
|
||||
Reference in New Issue
Block a user