refactor(go): make history totals opt in

2026-07-03 07:28:02 +08:00
parent bb0d001d72
commit b149660b4b
3 changed files with 104 additions and 24 deletions
--- a/docs/architecture/storage-minimal-contract.md
+++ b/docs/architecture/storage-minimal-contract.md
@@ -49,14 +49,19 @@
 4. `vehicle_locations` 不再接收临时身份兜底字段。
   - 原因：位置历史是正式车辆时间序列，只按 `protocol + vin` 分表和查询。
   - 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`，待 identity/binding 修复后可从 raw 重放补写。
+   - 查询：`/api/history/locations` 默认不执行大表 `COUNT(*)`，`total` 表示本页返回条数；只有需要精确总数时传 `includeTotal=true`。
   - 生产：ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol`、`vin` 两个 tag。

-5. MySQL realtime 当前态表不保留代理主键和创建时间。
+5. 历史 raw 查询默认不做精确总数。
+   - 原因：`raw_frames` 是最高写入量证据层，分页/排查通常只需要最新一页数据。
+   - 查询：`/api/history/raw-frames` 默认 `total` 为本页返回条数；只有导出前预估、后台管理需要总页数时才传 `includeTotal=true`。
+
+6. MySQL realtime 当前态表不保留代理主键和创建时间。
   - 原因：`vehicle_realtime_snapshot`、`vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影，业务主键就是 `(protocol, vin)`。
   - 状态：schema 使用 `PRIMARY KEY (protocol, vin)`，不保留自增 `id` 和 `created_at`，对外只暴露最新 `updated_at`。
   - 生产：项目上线前可直接重建这两张 MySQL 表，实时数据会从 Kafka 新消息继续投影。

-6. identity 层只保留两张表。
+7. identity 层只保留两张表。
   - `vehicle_identity_binding`：VIN 与 plate/phone/device_id 的映射，供 808 等协议反查 VIN。
   - `jt808_registration`：808 注册、鉴权、最新活跃和 VIN 匹配状态。
   - 状态：`vehicle_identity_binding` 使用 VIN 主键；`jt808_registration` 使用 phone 主键；两张表都不保留代理自增主键和 `created_at`。
--- a/go/vehicle-gateway/internal/history/query.go
+++ b/go/vehicle-gateway/internal/history/query.go
@@ -54,12 +54,13 @@ type RawFrameRow struct {
 }

 type LocationQuery struct {
-	Protocol string
-	VIN      string
-	DateFrom string
-	DateTo   string
-	Limit    int
-	Offset   int
+	Protocol     string
+	VIN          string
+	IncludeTotal bool
+	DateFrom     string
+	DateTo       string
+	Limit        int
+	Offset       int
 }

 type LocationRow struct {
@@ -591,16 +592,22 @@ func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		writeHistoryError(w, http.StatusBadRequest, err.Error())
 		return
 	}
-	total, err := h.repository.Count(r.Context(), query)
-	if err != nil {
-		writeHistoryError(w, http.StatusInternalServerError, err.Error())
-		return
+	var total int64
+	if query.IncludeTotal {
+		total, err = h.repository.Count(r.Context(), query)
+		if err != nil {
+			writeHistoryError(w, http.StatusInternalServerError, err.Error())
+			return
+		}
 	}
 	rows, err := h.repository.Query(r.Context(), query)
 	if err != nil {
 		writeHistoryError(w, http.StatusInternalServerError, err.Error())
 		return
 	}
+	if !query.IncludeTotal {
+		total = int64(len(rows))
+	}
 	w.Header().Set("Content-Type", "application/json")
 	_ = json.NewEncoder(w).Encode(map[string]any{
 		"items":  rows,
@@ -628,7 +635,7 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
 		DeviceID:     values.Get("deviceId"),
 		MessageID:    values.Get("messageId"),
 		OrderBy:      values.Get("orderBy"),
-		IncludeTotal: values.Get("includeTotal") != "false",
+		IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
 		DateFrom:     values.Get("dateFrom"),
 		DateTo:       values.Get("dateTo"),
 		Limit:        limit,
@@ -656,12 +663,13 @@ func parseLocationQuery(r *http.Request) (LocationQuery, error) {
 		return LocationQuery{}, err
 	}
 	query := LocationQuery{
-		Protocol: values.Get("protocol"),
-		VIN:      values.Get("vin"),
-		DateFrom: values.Get("dateFrom"),
-		DateTo:   values.Get("dateTo"),
-		Limit:    limit,
-		Offset:   offset,
+		Protocol:     values.Get("protocol"),
+		VIN:          values.Get("vin"),
+		IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
+		DateFrom:     values.Get("dateFrom"),
+		DateTo:       values.Get("dateTo"),
+		Limit:        limit,
+		Offset:       offset,
 	}
 	if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
 		return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
--- a/go/vehicle-gateway/internal/history/query_test.go
+++ b/go/vehicle-gateway/internal/history/query_test.go
@@ -122,7 +122,7 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
 		))

 	handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
-	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
+	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&includeTotal=true&limit=5", nil)
 	response := httptest.NewRecorder()

 	handler.ServeHTTP(response, request)
@@ -164,7 +164,7 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
 		))

 	handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
-	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
+	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&includeTotal=true&limit=1", nil)
 	response := httptest.NewRecorder()

 	handler.ServeHTTP(response, request)
@@ -220,14 +220,47 @@ func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
 	}
 }

+func TestRawFrameHandlerSkipsTotalCountByDefault(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock.New() error = %v", err)
+	}
+	defer db.Close()
+	mock.ExpectQuery("ORDER BY ts DESC LIMIT 1 OFFSET 0").
+		WillReturnRows(sqlmock.NewRows([]string{
+			"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
+			"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
+			"protocol", "vehicle_key", "vin", "phone", "device_id",
+		}).AddRow(
+			"2026-07-02 01:26:48", "go_frame", "event-5", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
+			63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`,
+			"OK", "", "115.231.168.135:22170", "JT808", "JT808:13307811254", "", "13307811254", "",
+		))
+
+	handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
+	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&limit=1", nil)
+	response := httptest.NewRecorder()
+
+	handler.ServeHTTP(response, request)
+
+	if response.Code != http.StatusOK {
+		t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
+	}
+	body := response.Body.String()
+	if !strings.Contains(body, `"total":1`) {
+		t.Fatalf("response should use page size as total when total count is not requested: %s", body)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("sql expectations: %v", err)
+	}
+}
+
 func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
 	db, mock, err := sqlmock.New()
 	if err != nil {
 		t.Fatalf("sqlmock.New() error = %v", err)
 	}
 	defer db.Close()
-	mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
-		WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
 	mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
 		WillReturnRows(sqlmock.NewRows([]string{
 			"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
@@ -282,7 +315,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
 		))

 	handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
-	request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil)
+	request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&includeTotal=true&limit=1", nil)
 	response := httptest.NewRecorder()

 	handler.ServeHTTP(response, request)
@@ -306,6 +339,40 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
 	}
 }

+func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("sqlmock.New() error = %v", err)
+	}
+	defer db.Close()
+	mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'").
+		WillReturnRows(sqlmock.NewRows([]string{
+			"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
+			"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
+		}).AddRow(
+			"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
+			121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
+			"JT808", "LKLG7C4E3NA774736",
+		))
+
+	handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
+	request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil)
+	response := httptest.NewRecorder()
+
+	handler.ServeHTTP(response, request)
+
+	if response.Code != http.StatusOK {
+		t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
+	}
+	body := response.Body.String()
+	if !strings.Contains(body, `"total":1`) {
+		t.Fatalf("response should use page size as total when total count is not requested: %s", body)
+	}
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Fatalf("sql expectations: %v", err)
+	}
+}
+
 func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
 	handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
 	request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)