shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: skip raw freshness count scan

Browse Source
This commit is contained in:
lingniu
2026-07-02 09:30:15 +08:00
parent 42de422935
commit cbb6f3b741
4 changed files with 73 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
go/vehicle-gateway/internal/history/query.go
View File

@@ -17,17 +17,18 @@ type Queryer interface {
}
type RawFrameQuery struct {
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
MessageID string
OrderBy string
DateFrom string
DateTo string
Limit int
Offset int
Protocol string
VehicleKey string
VIN string
Phone string
DeviceID string
MessageID string
OrderBy string
IncludeTotal bool
DateFrom string
DateTo string
Limit int
Offset int
}
type RawFrameRow struct {
@@ -614,16 +615,22 @@ func (h *RawFrameHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
writeHistoryError(w, http.StatusBadRequest, err.Error())
return
}
total, err := h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
var total int64
if query.IncludeTotal {
total, err = h.repository.Count(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
}
rows, err := h.repository.Query(r.Context(), query)
if err != nil {
writeHistoryError(w, http.StatusInternalServerError, err.Error())
return
}
if !query.IncludeTotal {
total = int64(len(rows))
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"items": rows,
@@ -710,17 +717,18 @@ func parseRawFrameQuery(r *http.Request) (RawFrameQuery, error) {
return RawFrameQuery{}, err
}
query := RawFrameQuery{
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
Protocol: values.Get("protocol"),
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Phone: values.Get("phone"),
DeviceID: values.Get("deviceId"),
MessageID: values.Get("messageId"),
OrderBy: values.Get("orderBy"),
IncludeTotal: values.Get("includeTotal") != "false",
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDateTime(query.DateFrom) || !validDateTime(query.DateTo) {
return RawFrameQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD or YYYY-MM-DD HH:mm:ss")

37
go/vehicle-gateway/internal/history/query_test.go
View File

@@ -135,6 +135,43 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
}
}
func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "fields_json", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
63, "7E0200", "", `{"header":{"message_id":"0x0200"}}`, `{"speed_kmh":0}`,
"OK", "", "115.231.168.135:22170", "JT808", "JT808:013307811254", "", "013307811254", "",
))
handler := NewRawFrameHandler(NewRawFrameRepository(db, "lingniu_vehicle_ts"))
request := httptest.NewRequest(http.MethodGet, "/api/history/raw-frames?protocol=JT808&orderBy=receivedAt&includeTotal=false&limit=1", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"total":1`, `"received_at":"2026-07-02 01:26:48"`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {

2
tools/go_native_prod_smoke.py
View File

@@ -319,7 +319,7 @@ def build_check_specs(
min_stat: int,
max_raw_age_minutes: float | None = None,
) -> list[CheckSpec]:
raw_params = {"limit": 1, "orderBy": "receivedAt"}
raw_params = {"limit": 1, "orderBy": "receivedAt", "includeTotal": "false"}
gb32960_history_params = {"protocol": "GB32960", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
jt808_history_params = {"protocol": "JT808", "dateFrom": date_from, "dateTo": date_to, "limit": 1}
yutong_mqtt_history_params = {"protocol": "YUTONG_MQTT", "dateFrom": date_from, "dateTo": date_to, "limit": 1}

1
tools/test_go_native_prod_smoke.py
View File

@@ -118,6 +118,7 @@ class GoNativeProdSmokeTest(unittest.TestCase):
self.assertTrue(raw_specs)
for spec in raw_specs:
self.assertEqual(spec.params.get("orderBy"), "receivedAt")
self.assertEqual(spec.params.get("includeTotal"), "false")
self.assertNotIn("dateFrom", spec.params)
self.assertNotIn("dateTo", spec.params)