refactor(go): make history totals opt in

This commit is contained in:
lingniu
2026-07-03 07:28:02 +08:00
parent bb0d001d72
commit b149660b4b
3 changed files with 104 additions and 24 deletions

View File

@@ -49,14 +49,19 @@
4. `vehicle_locations` 不再接收临时身份兜底字段。 4. `vehicle_locations` 不再接收临时身份兜底字段。
- 原因:位置历史是正式车辆时间序列,只按 `protocol + vin` 分表和查询。 - 原因:位置历史是正式车辆时间序列,只按 `protocol + vin` 分表和查询。
- 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。 - 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。
- 查询:`/api/history/locations` 默认不执行大表 `COUNT(*)``total` 表示本页返回条数;只有需要精确总数时传 `includeTotal=true`
- 生产ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol``vin` 两个 tag。 - 生产ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol``vin` 两个 tag。
5. MySQL realtime 当前态表不保留代理主键和创建时间 5. 历史 raw 查询默认不做精确总数
- 原因:`raw_frames` 是最高写入量证据层,分页/排查通常只需要最新一页数据。
- 查询:`/api/history/raw-frames` 默认 `total` 为本页返回条数;只有导出前预估、后台管理需要总页数时才传 `includeTotal=true`
6. MySQL realtime 当前态表不保留代理主键和创建时间。
- 原因:`vehicle_realtime_snapshot``vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影,业务主键就是 `(protocol, vin)` - 原因:`vehicle_realtime_snapshot``vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影,业务主键就是 `(protocol, vin)`
- 状态schema 使用 `PRIMARY KEY (protocol, vin)`,不保留自增 `id``created_at`,对外只暴露最新 `updated_at` - 状态schema 使用 `PRIMARY KEY (protocol, vin)`,不保留自增 `id``created_at`,对外只暴露最新 `updated_at`
- 生产:项目上线前可直接重建这两张 MySQL 表,实时数据会从 Kafka 新消息继续投影。 - 生产:项目上线前可直接重建这两张 MySQL 表,实时数据会从 Kafka 新消息继续投影。
6. identity 层只保留两张表。 7. identity 层只保留两张表。
- `vehicle_identity_binding`VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。 - `vehicle_identity_binding`VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。
- `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。 - `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。
- 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at` - 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at`

View File

@@ -54,12 +54,13 @@ type RawFrameRow struct {
} }
type LocationQuery struct { type LocationQuery struct {
Protocol string Protocol string
VIN string VIN string
DateFrom string IncludeTotal bool
DateTo string DateFrom string
Limit int DateTo string
Offset int Limit int
Offset int
} }
type LocationRow struct { type LocationRow struct {
@@ -591,16 +592,22 @@ func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeHistoryError(w, http.StatusBadRequest, err.Error()) writeHistoryError(w, http.StatusBadRequest, err.Error())
return return
} }
total, err := h.repository.Count(r.Context(), query) var total int64
if err != nil { if query.IncludeTotal {
writeHistoryError(w, http.StatusInternalServerError, err.Error()) total, err = h.repository.Count(r.Context(), query)
return if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
} }
rows, err := h.repository.Query(r.Context(), query) rows, err := h.repository.Query(r.Context(), query)
if err != nil { if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error()) writeHistoryError(w, http.StatusInternalServerError, err.Error())
return return
} }
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]any{
"items": rows, "items": rows,
@@ -628,7 +635,7 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
DeviceID: values.Get("deviceId"), DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"), MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"), OrderBy: values.Get("orderBy"),
IncludeTotal: values.Get("includeTotal") != "false", IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
DateFrom: values.Get("dateFrom"), DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"), DateTo: values.Get("dateTo"),
Limit: limit, Limit: limit,
@@ -656,12 +663,13 @@ func parseLocationQuery(r *http.Request) (LocationQuery, error) {
return LocationQuery{}, err return LocationQuery{}, err
} }
query := LocationQuery{ query := LocationQuery{
Protocol: values.Get("protocol"), Protocol: values.Get("protocol"),
VIN: values.Get("vin"), VIN: values.Get("vin"),
DateFrom: values.Get("dateFrom"), IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
DateTo: values.Get("dateTo"), DateFrom: values.Get("dateFrom"),
Limit: limit, DateTo: values.Get("dateTo"),
Offset: offset, Limit: limit,
Offset: offset,
} }
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) { if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss") return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")

View File

@@ -122,7 +122,7 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
)) ))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts")) handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil) request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&includeTotal=true&limit=5", nil)
response := httptest.NewRecorder() response := httptest.NewRecorder()
handler.ServeHTTP(response, request) handler.ServeHTTP(response, request)
@@ -164,7 +164,7 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
)) ))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts")) handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil) request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&includeTotal=true&limit=1", nil)
response := httptest.NewRecorder() response := httptest.NewRecorder()
handler.ServeHTTP(response, request) handler.ServeHTTP(response, request)
@@ -220,14 +220,47 @@ func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
} }
} }
func TestRawFrameHandlerSkipsTotalCountByDefault(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY ts DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-5", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:13307811254", "", "13307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"total":1`) {
t.Fatalf("response should use page size as total when total count is not requested: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) { func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New() db, mock, err := sqlmock.New()
if err != nil { if err != nil {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames"). mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes", "ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
@@ -282,7 +315,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
)) ))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts")) handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil) request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&includeTotal=true&limit=1", nil)
response := httptest.NewRecorder() response := httptest.NewRecorder()
handler.ServeHTTP(response, request) handler.ServeHTTP(response, request)
@@ -306,6 +339,40 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
} }
} }
func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
}).AddRow(
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
"JT808", "LKLG7C4E3NA774736",
))
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
if !strings.Contains(body, `"total":1`) {
t.Fatalf("response should use page size as total when total count is not requested: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) { func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, "")) handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil) request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)