refactor(go): make history totals opt in
This commit is contained in:
@@ -49,14 +49,19 @@
|
|||||||
4. `vehicle_locations` 不再接收临时身份兜底字段。
|
4. `vehicle_locations` 不再接收临时身份兜底字段。
|
||||||
- 原因:位置历史是正式车辆时间序列,只按 `protocol + vin` 分表和查询。
|
- 原因:位置历史是正式车辆时间序列,只按 `protocol + vin` 分表和查询。
|
||||||
- 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。
|
- 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。
|
||||||
|
- 查询:`/api/history/locations` 默认不执行大表 `COUNT(*)`,`total` 表示本页返回条数;只有需要精确总数时传 `includeTotal=true`。
|
||||||
- 生产:ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol`、`vin` 两个 tag。
|
- 生产:ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol`、`vin` 两个 tag。
|
||||||
|
|
||||||
5. MySQL realtime 当前态表不保留代理主键和创建时间。
|
5. 历史 raw 查询默认不做精确总数。
|
||||||
|
- 原因:`raw_frames` 是最高写入量证据层,分页/排查通常只需要最新一页数据。
|
||||||
|
- 查询:`/api/history/raw-frames` 默认 `total` 为本页返回条数;只有导出前预估、后台管理需要总页数时才传 `includeTotal=true`。
|
||||||
|
|
||||||
|
6. MySQL realtime 当前态表不保留代理主键和创建时间。
|
||||||
- 原因:`vehicle_realtime_snapshot`、`vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影,业务主键就是 `(protocol, vin)`。
|
- 原因:`vehicle_realtime_snapshot`、`vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影,业务主键就是 `(protocol, vin)`。
|
||||||
- 状态:schema 使用 `PRIMARY KEY (protocol, vin)`,不保留自增 `id` 和 `created_at`,对外只暴露最新 `updated_at`。
|
- 状态:schema 使用 `PRIMARY KEY (protocol, vin)`,不保留自增 `id` 和 `created_at`,对外只暴露最新 `updated_at`。
|
||||||
- 生产:项目上线前可直接重建这两张 MySQL 表,实时数据会从 Kafka 新消息继续投影。
|
- 生产:项目上线前可直接重建这两张 MySQL 表,实时数据会从 Kafka 新消息继续投影。
|
||||||
|
|
||||||
6. identity 层只保留两张表。
|
7. identity 层只保留两张表。
|
||||||
- `vehicle_identity_binding`:VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。
|
- `vehicle_identity_binding`:VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。
|
||||||
- `jt808_registration`:808 注册、鉴权、最新活跃和 VIN 匹配状态。
|
- `jt808_registration`:808 注册、鉴权、最新活跃和 VIN 匹配状态。
|
||||||
- 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at`。
|
- 状态:`vehicle_identity_binding` 使用 VIN 主键;`jt808_registration` 使用 phone 主键;两张表都不保留代理自增主键和 `created_at`。
|
||||||
|
|||||||
@@ -54,12 +54,13 @@ type RawFrameRow struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type LocationQuery struct {
|
type LocationQuery struct {
|
||||||
Protocol string
|
Protocol string
|
||||||
VIN string
|
VIN string
|
||||||
DateFrom string
|
IncludeTotal bool
|
||||||
DateTo string
|
DateFrom string
|
||||||
Limit int
|
DateTo string
|
||||||
Offset int
|
Limit int
|
||||||
|
Offset int
|
||||||
}
|
}
|
||||||
|
|
||||||
type LocationRow struct {
|
type LocationRow struct {
|
||||||
@@ -591,16 +592,22 @@ func (h *LocationHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
writeHistoryError(w, http.StatusBadRequest, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
total, err := h.repository.Count(r.Context(), query)
|
var total int64
|
||||||
if err != nil {
|
if query.IncludeTotal {
|
||||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
total, err = h.repository.Count(r.Context(), query)
|
||||||
return
|
if err != nil {
|
||||||
|
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
rows, err := h.repository.Query(r.Context(), query)
|
rows, err := h.repository.Query(r.Context(), query)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
writeHistoryError(w, http.StatusInternalServerError, err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if !query.IncludeTotal {
|
||||||
|
total = int64(len(rows))
|
||||||
|
}
|
||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
"items": rows,
|
"items": rows,
|
||||||
@@ -628,7 +635,7 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
|
|||||||
DeviceID: values.Get("deviceId"),
|
DeviceID: values.Get("deviceId"),
|
||||||
MessageID: values.Get("messageId"),
|
MessageID: values.Get("messageId"),
|
||||||
OrderBy: values.Get("orderBy"),
|
OrderBy: values.Get("orderBy"),
|
||||||
IncludeTotal: values.Get("includeTotal") != "false",
|
IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
|
||||||
DateFrom: values.Get("dateFrom"),
|
DateFrom: values.Get("dateFrom"),
|
||||||
DateTo: values.Get("dateTo"),
|
DateTo: values.Get("dateTo"),
|
||||||
Limit: limit,
|
Limit: limit,
|
||||||
@@ -656,12 +663,13 @@ func parseLocationQuery(r *http.Request) (LocationQuery, error) {
|
|||||||
return LocationQuery{}, err
|
return LocationQuery{}, err
|
||||||
}
|
}
|
||||||
query := LocationQuery{
|
query := LocationQuery{
|
||||||
Protocol: values.Get("protocol"),
|
Protocol: values.Get("protocol"),
|
||||||
VIN: values.Get("vin"),
|
VIN: values.Get("vin"),
|
||||||
DateFrom: values.Get("dateFrom"),
|
IncludeTotal: strings.EqualFold(strings.TrimSpace(values.Get("includeTotal")), "true"),
|
||||||
DateTo: values.Get("dateTo"),
|
DateFrom: values.Get("dateFrom"),
|
||||||
Limit: limit,
|
DateTo: values.Get("dateTo"),
|
||||||
Offset: offset,
|
Limit: limit,
|
||||||
|
Offset: offset,
|
||||||
}
|
}
|
||||||
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
|
||||||
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
return LocationQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")
|
||||||
|
|||||||
@@ -122,7 +122,7 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
|
|||||||
))
|
))
|
||||||
|
|
||||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&limit=5", nil)
|
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=GB32960&vin=LB9A32A21R0LS1707&includeTotal=true&limit=5", nil)
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
|
|
||||||
handler.ServeHTTP(response, request)
|
handler.ServeHTTP(response, request)
|
||||||
@@ -164,7 +164,7 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
|
|||||||
))
|
))
|
||||||
|
|
||||||
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&limit=1", nil)
|
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?vehicleKey=JT808:013307811350&protocol=JT808&includeTotal=true&limit=1", nil)
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
|
|
||||||
handler.ServeHTTP(response, request)
|
handler.ServeHTTP(response, request)
|
||||||
@@ -220,14 +220,47 @@ func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRawFrameHandlerSkipsTotalCountByDefault(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery("ORDER BY ts DESC LIMIT 1 OFFSET 0").
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
|
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||||
|
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
|
||||||
|
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||||
|
}).AddRow(
|
||||||
|
"2026-07-02 01:26:48", "go_frame", "event-5", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
|
||||||
|
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`,
|
||||||
|
"OK", "", "115.231.168.135:22170", "JT808", "JT808:13307811254", "", "13307811254", "",
|
||||||
|
))
|
||||||
|
|
||||||
|
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
|
||||||
|
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&limit=1", nil)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
|
||||||
|
handler.ServeHTTP(response, request)
|
||||||
|
|
||||||
|
if response.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||||
|
}
|
||||||
|
body := response.Body.String()
|
||||||
|
if !strings.Contains(body, `"total":1`) {
|
||||||
|
t.Fatalf("response should use page size as total when total count is not requested: %s", body)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("sql expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
|
||||||
db, mock, err := sqlmock.New()
|
db, mock, err := sqlmock.New()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("sqlmock.New() error = %v", err)
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
|
|
||||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
|
|
||||||
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
|
||||||
WillReturnRows(sqlmock.NewRows([]string{
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
|
||||||
@@ -282,7 +315,7 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
|
|||||||
))
|
))
|
||||||
|
|
||||||
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
||||||
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil)
|
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&includeTotal=true&limit=1", nil)
|
||||||
response := httptest.NewRecorder()
|
response := httptest.NewRecorder()
|
||||||
|
|
||||||
handler.ServeHTTP(response, request)
|
handler.ServeHTTP(response, request)
|
||||||
@@ -306,6 +339,40 @@ func TestLocationHandlerReturnsLocationsByVIN(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocationHandlerSkipsTotalCountByDefault(t *testing.T) {
|
||||||
|
db, mock, err := sqlmock.New()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
mock.ExpectQuery("vin = 'LKLG7C4E3NA774736'").
|
||||||
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
|
"ts", "event_id", "frame_id", "received_at", "longitude", "latitude", "altitude_m", "speed_kmh",
|
||||||
|
"direction_deg", "alarm_flag", "status_flag", "total_mileage_km", "protocol", "vin",
|
||||||
|
}).AddRow(
|
||||||
|
"2026-07-02 00:18:22", "event-3", "go_frame", "2026-07-02 00:22:43",
|
||||||
|
121.07764, 30.585928, 11.0, 8.0, 171, 0, 4718595, 8792.8,
|
||||||
|
"JT808", "LKLG7C4E3NA774736",
|
||||||
|
))
|
||||||
|
|
||||||
|
handler := NewLocationHandler(NewLocationRepository(db, "lingniu_vehicle_ts"))
|
||||||
|
request := httptest.NewRequest(http.MethodGet, "/api/history/locations?vin=LKLG7C4E3NA774736&protocol=JT808&limit=1", nil)
|
||||||
|
response := httptest.NewRecorder()
|
||||||
|
|
||||||
|
handler.ServeHTTP(response, request)
|
||||||
|
|
||||||
|
if response.Code != http.StatusOK {
|
||||||
|
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
|
||||||
|
}
|
||||||
|
body := response.Body.String()
|
||||||
|
if !strings.Contains(body, `"total":1`) {
|
||||||
|
t.Fatalf("response should use page size as total when total count is not requested: %s", body)
|
||||||
|
}
|
||||||
|
if err := mock.ExpectationsWereMet(); err != nil {
|
||||||
|
t.Fatalf("sql expectations: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
|
func TestRawFrameHandlerRejectsInvalidLimit(t *testing.T) {
|
||||||
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
|
handler := NewRawFrameHandler(NewRawFrameRepository(&sql.DB{}, ""))
|
||||||
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
|
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?limit=501", nil)
|
||||||
|
|||||||
Reference in New Issue
Block a user