From b149660b4b410518acc4a02d3addb765ecd2deff Mon Sep 17 00:00:00 2001 From: lingniu Date: Fri, 3 Jul 2026 07:28:02 +0800 Subject: [PATCH] refactor(go): make history totals opt in --- docs/architecture/storage-minimal-contract.md | 9 ++- go/vehicle-gateway/internal/history/query.go | 42 ++++++---- .../internal/history/query_test.go | 77 +++++++++++++++++-- 3 files changed, 104 insertions(+), 24 deletions(-) diff --git a/docs/architecture/storage-minimal-contract.md b/docs/architecture/storage-minimal-contract.md index 49ad767a..e221db67 100644 --- a/docs/architecture/storage-minimal-contract.md +++ b/docs/architecture/storage-minimal-contract.md @@ -49,14 +49,19 @@ 4. `vehicle_locations` 不再接收临时身份兜底字段。 - 原因：位置历史是正式车辆时间序列，只按 `protocol + vin` 分表和查询。 - 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`，待 identity/binding 修复后可从 raw 重放补写。 + - 查询：`/api/history/locations` 默认不执行大表 `COUNT(*)`，`total` 表示本页返回条数；只有需要精确总数时传 `includeTotal=true`。 - 生产：ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol`、`vin` 两个 tag。 -5. MySQL realtime 当前态表不保留代理主键和创建时间。 +5. 历史 raw 查询默认不做精确总数。 + - 原因：`raw_frames` 是最高写入量证据层，分页/排查通常只需要最新一页数据。 + - 查询：`/api/history/raw-frames` 默认 `total` 为本页返回条数；只有导出前预估、后台管理需要总页数时才传 `includeTotal=true`。 + +6. MySQL realtime 当前态表不保留代理主键和创建时间。 - 原因：`vehicle_realtime_snapshot`、`vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影，业务主键就是 `(protocol, vin)`。 - 状态：schema 使用 `PRIMARY KEY (protocol, vin)`，不保留自增 `id` 和 `created_at`，对外只暴露最新 `updated_at`。 - 生产：项目上线前可直接重建这两张 MySQL 表，实时数据会从 Kafka 新消息继续投影。 -6. identity 层只保留两张表。 +7. identity 层只保留两张表。 - `vehicle_identity_binding`：VIN 与 plate/phone/device_id 的映射，供 808 等协议反查 VIN。 - `jt808_registration`：808 注册、鉴权、最新活跃和 VIN 匹配状态。 - 状态：`vehicle_identity_binding` 使用 VIN 主键；`jt808_registration` 使用 phone 主键；两张表都不保留代理自增主键和 `created_at`。 diff --git a/go/vehicle-gateway/internal/history/query.go b/go/vehicle-gateway/internal/history/query.go index f6d57b8d..fe1038f1 100644 --- a/go/vehicle-gateway/internal/history/query.go +++ b/go/vehicle-gateway/internal/history/query.go @@ -54,12 +54,13 @@ type RawFrameRow struct { } type LocationQuery struct { - Protocol string - VIN string - DateFrom string - DateTo string - Limit int - Offset int + Protocol string + VIN string + IncludeTotal bool + DateFrom string + DateTo string + Limit int + Offset int } type LocationRow struct { @@ -591,16 +592,22 @@ func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { writeHistoryError(w, http.StatusBadRequest, err.Error()) return } - total, err := h.repository.Count(r.Context(), query) - if err != nil { - writeHistoryError(w, http.StatusInternalServerError, err.Error()) - return + var total int64 + if query.IncludeTotal { + total, err = h.repository.Count(r.Context(), query) + if err != nil { + writeHistoryError(w, http.StatusInternalServerError, err.Error()) + return + } } rows, err := h.repository.Query(r.Context(), query) if err != nil { writeHistoryError(w, http.StatusInternalServerError, err.Error()) return } + if !query.IncludeTotal { + total = int64(len(rows)) + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "items": rows, @@ -628,7 +635,7 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) { DeviceID: values.Get("deviceId"), MessageID: values.Get("messageId"), OrderBy: values.Get("orderBy"), - IncludeTotal: values.Get("includeTotal") != "false", + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), DateFrom: values.Get("dateFrom"), DateTo: values.Get("dateTo"), Limit: limit, @@ -656,12 +663,13 @@ func parseLocationQuery(r *http.Request) (LocationQuery, error) { return LocationQuery{}, err } query := LocationQuery{ - Protocol: values.Get("protocol"), - VIN: values.Get("vin"), - DateFrom: values.Get("dateFrom"), - DateTo: values.Get("dateTo"), - Limit: limit, - Offset: offset, + Protocol: values.Get("protocol"), + VIN: values.Get("vin"), + IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"), + DateFrom: values.Get("dateFrom"), + DateTo: values.Get("dateTo"), + Limit: limit, + Offset: offset, } if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) { return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss") diff --git a/go/vehicle-gateway/internal/history/query_test.go b/go/vehicle-gateway/internal/history/query_test.go index faf7000f..01bcd7ea 100644 --- a/go/vehicle-gateway/internal/history/query_test.go +++ b/go/vehicle-gateway/internal/history/query_test.go @@ -122,7 +122,7 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) { )) handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts")) - request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil) + request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&includeTotal=true&limit=5", nil) response := httptest.NewRecorder() handler.ServeHTTP(response, request) @@ -164,7 +164,7 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) { )) handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts")) - request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil) + request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&includeTotal=true&limit=1", nil) response := httptest.NewRecorder() handler.ServeHTTP(response, request) @@ -220,14 +220,47 @@ func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) { } } +func TestRawFrameHandlerSkipsTotalCountByDefault(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("ORDER BY ts DESC LIMIT 1 OFFSET 0"). + WillReturnRows(sqlmock.NewRows([]string{ + "ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes", + "raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint", + "protocol", "vehicle_key", "vin", "phone", "device_id", + }).AddRow( + "2026-07-02 01:26:48", "go_frame", "event-5", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48", + 63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, + "OK", "", "115.231.168.135:22170", "JT808", "JT808:13307811254", "", "13307811254", "", + )) + + handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts")) + request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&limit=1", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + if !strings.Contains(body, `"total":1`) { + t.Fatalf("response should use page size as total when total count is not requested: %s", body) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) } defer db.Close() - mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames"). - WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0)) mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames"). WillReturnRows(sqlmock.NewRows([]string{ "ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes", @@ -282,7 +315,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) { )) handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts")) - request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil) + request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&includeTotal=true&limit=1", nil) response := httptest.NewRecorder() handler.ServeHTTP(response, request) @@ -306,6 +339,40 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) { } } +func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'"). + WillReturnRows(sqlmock.NewRows([]string{ + "ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh", + "direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin", + }).AddRow( + "2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43", + 121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8, + "JT808", "LKLG7C4E3NA774736", + )) + + handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts")) + request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + if !strings.Contains(body, `"total":1`) { + t.Fatalf("response should use page size as total when total count is not requested: %s", body) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("sql expectations: %v", err) + } +} + func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) { handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, "")) request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)