shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf(go): optimize tdengine raw frame queries

Browse Source
This commit is contained in:
lingniu
2026-07-03 14:25:48 +08:00
parent 535acdb2a6
commit 6318c10c41
4 changed files with 123 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
go/vehicle-gateway/internal/history/query_test.go
View File

@@ -19,10 +19,10 @@ func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, .* AS parsed_fields, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_jt808_").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
time.Date(2026, 7, 1, 23, 25, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
@@ -35,11 +35,12 @@ func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
rows, err := repository.Query(context.Background(), RawFrameQuery{
Protocol: "JT808",
VIN: "LKLG7C4E3NA774736",
DateFrom: "2026-07-01 00:00:00",
DateTo: "2026-07-01 23:59:59",
Limit: 20,
Protocol: "JT808",
VIN: "LKLG7C4E3NA774736",
DateFrom: "2026-07-01 00:00:00",
DateTo: "2026-07-01 23:59:59",
IncludeFields: true,
Limit: 20,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
@@ -47,7 +48,7 @@ func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedJSON == "" {
if rows[0].MessageID != 512 || rows[0].MessageIDHex != "0x0200" || rows[0].ParsedFields == "" {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].TS != "2026-07-01 23:25:36" {
@@ -58,18 +59,18 @@ func TestRawFrameRepositoryQueriesRawFramesWithFilters(t *testing.T) {
}
}
func TestRawFrameRepositoryHydratesChunkedParsedJSON(t *testing.T) {
func TestRawFrameRepositoryHydratesChunkedParsedFields(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
manifest := `{"chunked":true,"payload_kind":"parsed_json","event_id":"event-oversized","chunk_count":2}`
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
manifest := `{"chunked":true,"payload_kind":"parsed_fields","event_id":"event-oversized","chunk_count":2}`
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, .* AS parsed_fields, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_gb32960_").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 10:00:00", "go_frame", "event-oversized", 2,
@@ -79,14 +80,15 @@ func TestRawFrameRepositoryHydratesChunkedParsedJSON(t *testing.T) {
))
mock.ExpectQuery("SELECT event_id, payload_kind, chunk_index, chunk_text FROM lingniu_vehicle_ts.raw_frame_payload_chunks").
WillReturnRows(sqlmock.NewRows([]string{"event_id", "payload_kind", "chunk_index", "chunk_text"}).
AddRow("event-oversized", "parsed_json", 0, `{"data_units":[`).
AddRow("event-oversized", "parsed_json", 1, `{"field":"value"}]}`))
AddRow("event-oversized", "parsed_fields", 0, `{"data_units":[`).
AddRow("event-oversized", "parsed_fields", 1, `{"field":"value"}]}`))
repository := NewRawFrameRepository(db, "lingniu_vehicle_ts")
rows, err := repository.Query(context.Background(), RawFrameQuery{
Protocol: "GB32960",
VIN: "LB9A32A21R0LS1707",
Limit: 1,
Protocol: "GB32960",
VIN: "LB9A32A21R0LS1707",
IncludeFields: true,
Limit: 1,
})
if err != nil {
t.Fatalf("Query() error = %v", err)
@@ -94,8 +96,8 @@ func TestRawFrameRepositoryHydratesChunkedParsedJSON(t *testing.T) {
if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows))
}
if want := `{"data_units":[{"field":"value"}]}`; rows[0].ParsedJSON != want {
t.Fatalf("hydrated parsed_json = %q, want %q", rows[0].ParsedJSON, want)
if want := `{"data_units":[{"field":"value"}]}`; rows[0].ParsedFields != want {
t.Fatalf("hydrated parsed_fields = %q, want %q", rows[0].ParsedFields, want)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
@@ -108,12 +110,12 @@ func TestRawFrameHandlerReturnsRawFrames(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_gb32960_").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(38))
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, .* AS parsed_fields, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_gb32960_").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-01 22:28:25", "go_frame", "event-2", 2, "2026-07-01 22:28:25", "2026-07-01 22:28:25",
@@ -150,12 +152,12 @@ func TestRawFrameHandlerFiltersByVehicleKey(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_frames").
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.raw_jt808_").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(11))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 00:18:22", "go_frame", "event-3", 0x0200, "2026-07-02 00:18:22", "2026-07-02 00:22:43",
@@ -192,7 +194,7 @@ func TestRawFrameHandlerCanSkipTotalCountForFreshnessProbe(t *testing.T) {
mock.ExpectQuery("ORDER BY received_at DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-4", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
@@ -229,7 +231,7 @@ func TestRawFrameHandlerSkipsTotalCountByDefault(t *testing.T) {
mock.ExpectQuery("ORDER BY ts DESC LIMIT 1 OFFSET 0").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}).AddRow(
"2026-07-02 01:26:48", "go_frame", "event-5", 0x0200, "2026-07-02 01:20:46", "2026-07-02 01:26:48",
@@ -261,10 +263,10 @@ func TestRawFrameHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, raw_hex, raw_text, parsed_json, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
mock.ExpectQuery("SELECT ts, frame_id, event_id, message_id, event_time, received_at, raw_size_bytes, .* AS parsed_fields, parse_status, parse_error, source_endpoint, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.raw_frames").
WillReturnRows(sqlmock.NewRows([]string{
"ts", "frame_id", "event_id", "message_id", "event_time", "received_at", "raw_size_bytes",
"raw_hex", "raw_text", "parsed_json", "parse_status", "parse_error", "source_endpoint",
"raw_hex", "raw_text", "parsed_fields", "parse_status", "parse_error", "source_endpoint",
"protocol", "vehicle_key", "vin", "phone", "device_id",
}))