perf: skip raw freshness count scan

This commit is contained in:
lingniu
2026-07-02 09:30:15 +08:00
parent 42de422935
commit cbb6f3b741
4 changed files with 73 additions and 27 deletions

View File

@@ -17,17 +17,18 @@ type Queryer interface {
} }
type RawFrameQuery struct { type RawFrameQuery struct {
Protocol string Protocol string
VehicleKey string VehicleKey string
VIN string VIN string
Phone string Phone string
DeviceID string DeviceID string
MessageID string MessageID string
OrderBy string OrderBy string
DateFrom string IncludeTotal bool
DateTo string DateFrom string
Limit int DateTo string
Offset int Limit int
Offset int
} }
type RawFrameRow struct { type RawFrameRow struct {
@@ -614,16 +615,22 @@ func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeHistoryError(w, http.StatusBadRequest, err.Error()) writeHistoryError(w, http.StatusBadRequest, err.Error())
return return
} }
total, err := h.repository.Count(r.Context(), query) var total int64
if err != nil { if query.IncludeTotal {
writeHistoryError(w, http.StatusInternalServerError, err.Error()) total, err = h.repository.Count(r.Context(), query)
return if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
} }
rows, err := h.repository.Query(r.Context(), query) rows, err := h.repository.Query(r.Context(), query)
if err != nil { if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error()) writeHistoryError(w, http.StatusInternalServerError, err.Error())
return return
} }
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{ _ = json.NewEncoder(w).Encode(map[string]any{
"items": rows, "items": rows,
@@ -710,17 +717,18 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
return RawFrameQuery{}, err return RawFrameQuery{}, err
} }
query := RawFrameQuery{ query := RawFrameQuery{
Protocol: values.Get("protocol"), Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"), VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"), VIN: values.Get("vin"),
Phone: values.Get("phone"), Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"), DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"), MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"), OrderBy: values.Get("orderBy"),
DateFrom: values.Get("dateFrom"), IncludeTotal: values.Get("includeTotal") != "false",
DateTo: values.Get("dateTo"), DateFrom: values.Get("dateFrom"),
Limit: limit, DateTo: values.Get("dateTo"),
Offset: offset, Limit: limit,
Offset: offset,
} }
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) { if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss") return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")

View File

@@ -135,6 +135,43 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
} }
} }
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) { func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New() db, mock, err := sqlmock.New()
if err != nil { if err != nil {

View File

@@ -319,7 +319,7 @@ def build_check_specs(
min_stat: int, min_stat: int,
max_raw_age_minutes: float | None = None, max_raw_age_minutes: float | None = None,
) -> list[CheckSpec]: ) -> list[CheckSpec]:
raw_params = {"limit": 1, "orderBy": "receivedAt"} raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1} gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1} jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1} yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}

View File

@@ -118,6 +118,7 @@ class GoNativeProdSmokeTest(unittest.TestCase):
self.assertTrue(raw_specs) self.assertTrue(raw_specs)
for spec in raw_specs: for spec in raw_specs:
self.assertEqual(spec.params.get("orderBy"), "receivedAt") self.assertEqual(spec.params.get("orderBy"), "receivedAt")
self.assertEqual(spec.params.get("includeTotal"), "false")
self.assertNotIn("dateFrom", spec.params) self.assertNotIn("dateFrom", spec.params)
self.assertNotIn("dateTo", spec.params) self.assertNotIn("dateTo", spec.params)