refactor(go): simplify daily mileage storage

This commit is contained in:
lingniu
2026-07-02 20:01:03 +08:00
parent abc1b9870a
commit 593dc08870
7 changed files with 74 additions and 203 deletions

View File

@@ -33,7 +33,7 @@ flowchart LR
HW --> TDRAW["TDengine raw_frames"] HW --> TDRAW["TDengine raw_frames"]
HW --> TDLOC["TDengine vehicle_locations"] HW --> TDLOC["TDengine vehicle_locations"]
SW --> MYMET["MySQL vehicle_daily_metric"] SW --> MYMET["MySQL vehicle_daily_mileage"]
RA --> REDIS["Redis realtime-raw"] RA --> REDIS["Redis realtime-raw"]
RA --> MYSNAP["MySQL vehicle_realtime_snapshot"] RA --> MYSNAP["MySQL vehicle_realtime_snapshot"]
RA --> MYLOC["MySQL vehicle_realtime_location"] RA --> MYLOC["MySQL vehicle_realtime_location"]
@@ -49,7 +49,7 @@ flowchart LR
| TDengine | `vehicle_locations` | High-volume location history | time, vin, protocol, longitude, latitude, speed, direction, mileage | full parsed JSON | | TDengine | `vehicle_locations` | High-volume location history | time, vin, protocol, longitude, latitude, speed, direction, mileage | full parsed JSON |
| MySQL | `vehicle_realtime_snapshot` | Latest per-protocol vehicle heartbeat | protocol, vin, plate, event time, received time | phone, device, message sequence, parsed JSON | | MySQL | `vehicle_realtime_snapshot` | Latest per-protocol vehicle heartbeat | protocol, vin, plate, event time, received time | phone, device, message sequence, parsed JSON |
| MySQL | `vehicle_realtime_location` | Latest business location cache | vin, plate, location, speed, mileage, SOC, event time | full raw payload and protocol internals | | MySQL | `vehicle_realtime_location` | Latest business location cache | vin, plate, location, speed, mileage, SOC, event time | full raw payload and protocol internals |
| MySQL | `vehicle_daily_metric` | Queryable daily metrics | vin or vehicle key, date, metric key, value, method, source mileage | per-frame raw details | | MySQL | `vehicle_daily_mileage` | Queryable daily mileage | vin or vehicle key, date, protocol, daily mileage, source mileage | generic metric key/value rows, per-frame raw details |
| MySQL | `vehicle_identity_binding` | Manual identity mapping | vin, plate, phone, device id | registration history | | MySQL | `vehicle_identity_binding` | Manual identity mapping | vin, plate, phone, device id | registration history |
| MySQL | `jt808_registration` | JT808 registration and auth trace | phone, device id, plate, auth code, vin match state, first/latest seen | GB32960 or MQTT records | | MySQL | `jt808_registration` | JT808 registration and auth trace | phone, device id, plate, auth code, vin match state, first/latest seen | GB32960 or MQTT records |
| Redis | `realtime-raw:{protocol}:{vin}` | Latest full realtime protocol state | merged latest protocol payload | historical data | | Redis | `realtime-raw:{protocol}:{vin}` | Latest full realtime protocol state | merged latest protocol payload | historical data |

View File

@@ -23,7 +23,7 @@
| TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息 | | TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息 |
| MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON | | MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON |
| MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 | | MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 |
| MySQL | `vehicle_daily_metric` | 日期、VIN/临时 vehicle key、协议、指标名、指标值、首末总里程、样本数 | 每帧细节、位置点列表 | | MySQL | `vehicle_daily_mileage` | 日期、VIN/临时 vehicle key、协议、日里程、首末总里程、样本数 | 泛化 metric key/value、每帧细节、位置点列表 |
| MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、device_id 映射 | 注册历史 | | MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、device_id 映射 | 注册历史 |
| MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态 | GB32960/MQTT 注册信息 | | MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态 | GB32960/MQTT 注册信息 |
| Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议最新完整 parsed 状态 | 历史数据、统计结果 | | Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议最新完整 parsed 状态 | 历史数据、统计结果 |
@@ -40,9 +40,10 @@
- 兼容raw 查询只选择公共列,不依赖 `fields_json`;旧表保留该列也不影响查询。 - 兼容raw 查询只选择公共列,不依赖 `fields_json`;旧表保留该列也不影响查询。
- 生产ECS TDengine 历史库已在上线前重建,`raw_frames` 不再包含 `fields_json` - 生产ECS TDengine 历史库已在上线前重建,`raw_frames` 不再包含 `fields_json`
3. `vehicle_daily_metric``vehicle_key``vin` 同时存在。 3. `vehicle_daily_mileage``vehicle_key``vin` 同时存在。
- 当前保留原因JT808 可能先用 phone 作为临时 key后续才绑定 VIN。 - 当前保留原因JT808 可能先用 phone 作为临时 key后续才绑定 VIN。
- 收敛方向:业务查询以 VIN 为主;没有 VIN 的数据只用于排查和待绑定,不作为正式车辆指标。 - 收敛方向:业务查询以 VIN 为主;没有 VIN 的数据只用于排查和待绑定,不作为正式车辆指标。
- 生产RDS 已在上线前删除泛化 `vehicle_daily_metric`,改为专用 `vehicle_daily_mileage`
## 字段提升规则 ## 字段提升规则

View File

@@ -11,11 +11,6 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
) )
const (
MetricDailyMileageKM = "daily_mileage_km"
MetricDailyTotalMileageKM = "daily_total_mileage_km"
)
type Execer interface { type Execer interface {
ExecContext(context.Context, string, ...any) (sql.Result, error) ExecContext(context.Context, string, ...any) (sql.Result, error)
} }
@@ -30,8 +25,6 @@ type MetricSample struct {
VIN string VIN string
Protocol envelope.Protocol Protocol envelope.Protocol
StatDate string StatDate string
MetricKey string
MetricValue float64
TotalMileageKM float64 TotalMileageKM float64
} }
@@ -46,12 +39,9 @@ func NewWriter(exec Execer, loc *time.Location) *Writer {
} }
func (w *Writer) EnsureSchema(ctx context.Context) error { func (w *Writer) EnsureSchema(ctx context.Context) error {
if _, err := w.exec.ExecContext(ctx, DailyMetricTableSQL); err != nil { if _, err := w.exec.ExecContext(ctx, DailyMileageTableSQL); err != nil {
return err return err
} }
if db, ok := w.exec.(metadataDB); ok {
return migrateDailyMetricTable(ctx, db)
}
return nil return nil
} }
@@ -61,13 +51,11 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
return err return err
} }
for _, sample := range samples { for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMetricSQL, if _, err := w.exec.ExecContext(ctx, upsertDailyMileageSQL,
sample.VehicleKey, sample.VehicleKey,
sample.VIN, sample.VIN,
sample.StatDate, sample.StatDate,
string(sample.Protocol), string(sample.Protocol),
sample.MetricKey,
sample.MetricValue,
sample.TotalMileageKM, sample.TotalMileageKM,
sample.TotalMileageKM); err != nil { sample.TotalMileageKM); err != nil {
return err return err
@@ -100,33 +88,20 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
return nil, errors.New("event or received time is required") return nil, errors.New("event or received time is required")
} }
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02") statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{ return []MetricSample{{
{
VehicleKey: vehicleKey, VehicleKey: vehicleKey,
VIN: vin, VIN: vin,
Protocol: env.Protocol, Protocol: env.Protocol,
StatDate: statDate, StatDate: statDate,
MetricKey: MetricDailyMileageKM,
MetricValue: 0,
TotalMileageKM: totalMileage, TotalMileageKM: totalMileage,
}, }}, nil
{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
MetricKey: MetricDailyTotalMileageKM,
MetricValue: totalMileage,
TotalMileageKM: totalMileage,
},
}, nil
} }
const upsertDailyMetricSQL = ` const upsertDailyMileageSQL = `
INSERT INTO vehicle_daily_metric INSERT INTO vehicle_daily_mileage
(vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, (vehicle_key, vin, stat_date, protocol, daily_mileage_km,
first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method) first_total_mileage_km, latest_total_mileage_km, sample_count)
VALUES (?, ?, ?, ?, ?, ?, 'km', ?, ?, 1, 'TOTAL_MILEAGE_DIFF') VALUES (?, ?, ?, ?, 0, ?, ?, 1)
ON DUPLICATE KEY UPDATE ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin), vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE first_total_mileage_km = CASE
@@ -139,112 +114,22 @@ ON DUPLICATE KEY UPDATE
THEN VALUES(latest_total_mileage_km) THEN VALUES(latest_total_mileage_km)
ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km)) ELSE GREATEST(latest_total_mileage_km, VALUES(latest_total_mileage_km))
END, END,
metric_value = CASE daily_mileage_km = GREATEST(
WHEN metric_key = 'daily_mileage_km'
THEN GREATEST(
CASE CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0 WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km) THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km ELSE latest_total_mileage_km
END, END,
VALUES(latest_total_mileage_km) VALUES(latest_total_mileage_km)
) ) - CASE
- CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0 WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km) THEN VALUES(first_total_mileage_km)
ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km)) ELSE LEAST(first_total_mileage_km, VALUES(first_total_mileage_km))
END
WHEN metric_key = 'daily_total_mileage_km'
THEN GREATEST(
CASE
WHEN latest_total_mileage_km IS NULL OR latest_total_mileage_km <= 0
THEN VALUES(latest_total_mileage_km)
ELSE latest_total_mileage_km
END,
VALUES(latest_total_mileage_km)
)
ELSE VALUES(metric_value)
END, END,
sample_count = sample_count + 1, sample_count = sample_count + 1,
updated_at = CURRENT_TIMESTAMP updated_at = CURRENT_TIMESTAMP
` `
type metadataDB interface {
Execer
QueryRowContext(context.Context, string, ...any) *sql.Row
}
func migrateDailyMetricTable(ctx context.Context, db metadataDB) error {
columnExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND column_name = 'vehicle_key'`)
if err != nil {
return err
}
if !columnExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD COLUMN vehicle_key VARCHAR(96) NOT NULL DEFAULT '' AFTER id`); err != nil {
return err
}
}
if _, err := db.ExecContext(ctx, `UPDATE vehicle_daily_metric SET vehicle_key = vin WHERE vehicle_key = ''`); err != nil {
return err
}
vehicleIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric_vehicle'`)
if err != nil {
return err
}
if !vehicleIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key)`); err != nil {
return err
}
}
oldIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'uk_daily_metric'`)
if err != nil {
return err
}
if oldIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric DROP INDEX uk_daily_metric`); err != nil {
return err
}
}
vinIndexExists, err := informationSchemaExists(ctx, db, `
SELECT COUNT(*)
FROM information_schema.statistics
WHERE table_schema = DATABASE()
AND table_name = 'vehicle_daily_metric'
AND index_name = 'idx_vin'`)
if err != nil {
return err
}
if !vinIndexExists {
if _, err := db.ExecContext(ctx, `ALTER TABLE vehicle_daily_metric ADD KEY idx_vin (vin)`); err != nil {
return err
}
}
return nil
}
func informationSchemaExists(ctx context.Context, db metadataDB, query string) (bool, error) {
var count int
if err := db.QueryRowContext(ctx, query).Scan(&count); err != nil {
return false, err
}
return count > 0, nil
}
func floatField(env envelope.FrameEnvelope, key string) (float64, bool) { func floatField(env envelope.FrameEnvelope, key string) (float64, bool) {
if env.Fields == nil { if env.Fields == nil {
return 0, false return 0, false

View File

@@ -10,7 +10,7 @@ import (
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
) )
func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) { func TestSamplesFromEnvelopeDerivesDailyMileageSample(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600) loc := time.FixedZone("Asia/Shanghai", 8*3600)
env := envelope.FrameEnvelope{ env := envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808, Protocol: envelope.ProtocolJT808,
@@ -25,17 +25,14 @@ func TestSamplesFromEnvelopeDerivesDailyMileageAndTotal(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err) t.Fatalf("SamplesFromEnvelope() error = %v", err)
} }
if len(samples) != 2 { if len(samples) != 1 {
t.Fatalf("sample count = %d", len(samples)) t.Fatalf("sample count = %d", len(samples))
} }
if samples[0].MetricKey != MetricDailyMileageKM || samples[0].MetricValue != 0 {
t.Fatalf("unexpected daily mileage sample: %#v", samples[0])
}
if samples[0].VehicleKey != "LNBVIN00000000001" { if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey) t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
} }
if samples[1].MetricKey != MetricDailyTotalMileageKM || samples[1].MetricValue != 10241.2 { if samples[0].TotalMileageKM != 10241.2 {
t.Fatalf("unexpected daily total sample: %#v", samples[1]) t.Fatalf("total mileage = %v", samples[0].TotalMileageKM)
} }
if samples[0].StatDate != "2026-07-01" { if samples[0].StatDate != "2026-07-01" {
t.Fatalf("stat date = %q", samples[0].StatDate) t.Fatalf("stat date = %q", samples[0].StatDate)
@@ -55,7 +52,7 @@ func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err) t.Fatalf("SamplesFromEnvelope() error = %v", err)
} }
if len(samples) != 2 { if len(samples) != 1 {
t.Fatalf("sample count = %d", len(samples)) t.Fatalf("sample count = %d", len(samples))
} }
if samples[0].VehicleKey != "JT808:13307811254" { if samples[0].VehicleKey != "JT808:13307811254" {
@@ -110,7 +107,7 @@ func TestSamplesFromEnvelopeSkipsNonPositiveMileage(t *testing.T) {
} }
} }
func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) { func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
exec := &recordingExec{} exec := &recordingExec{}
writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600)) writer := NewWriter(exec, time.FixedZone("Asia/Shanghai", 8*3600))
if err := writer.EnsureSchema(context.Background()); err != nil { if err := writer.EnsureSchema(context.Background()); err != nil {
@@ -127,18 +124,21 @@ func TestWriterEnsuresSchemaAndUpsertsTwoMetrics(t *testing.T) {
t.Fatalf("Append() error = %v", err) t.Fatalf("Append() error = %v", err)
} }
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_metric") { if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_mileage") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query) t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
} }
if !strings.Contains(exec.calls[0].query, "vehicle_key") { if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query) t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
} }
if len(exec.calls) != 3 { if len(exec.calls) != 2 {
t.Fatalf("exec calls = %d", len(exec.calls)) t.Fatalf("exec calls = %d", len(exec.calls))
} }
if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") { if !strings.Contains(exec.calls[1].query, "ON DUPLICATE KEY UPDATE") {
t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query) t.Fatalf("unexpected upsert sql: %s", exec.calls[1].query)
} }
if strings.Contains(exec.calls[1].query, "metric_key") || strings.Contains(exec.calls[1].query, "metric_unit") {
t.Fatalf("daily mileage upsert should not use generic metric columns: %s", exec.calls[1].query)
}
if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") { if !strings.Contains(exec.calls[1].query, "first_total_mileage_km <= 0") {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query) t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
} }

View File

@@ -20,7 +20,6 @@ type MetricQuery struct {
VehicleKey string VehicleKey string
VIN string VIN string
Protocol string Protocol string
MetricKey string
DateFrom string DateFrom string
DateTo string DateTo string
Limit int Limit int
@@ -32,13 +31,10 @@ type MetricRow struct {
VIN string `json:"vin"` VIN string `json:"vin"`
StatDate string `json:"stat_date"` StatDate string `json:"stat_date"`
Protocol string `json:"protocol"` Protocol string `json:"protocol"`
MetricKey string `json:"metric_key"` DailyMileageKM float64 `json:"daily_mileage_km"`
MetricValue float64 `json:"metric_value"`
MetricUnit string `json:"metric_unit"`
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"` FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
SampleCount int64 `json:"sample_count"` SampleCount int64 `json:"sample_count"`
CalculationMethod string `json:"calculation_method"`
CreatedAt string `json:"created_at"` CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
} }
@@ -76,13 +72,10 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
&row.VIN, &row.VIN,
&statDate, &statDate,
&row.Protocol, &row.Protocol,
&row.MetricKey, &row.DailyMileageKM,
&row.MetricValue,
&row.MetricUnit,
&first, &first,
&latest, &latest,
&row.SampleCount, &row.SampleCount,
&row.CalculationMethod,
&createdAt, &createdAt,
&updatedAt, &updatedAt,
); err != nil { ); err != nil {
@@ -124,7 +117,6 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey) query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN) query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol)) query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.MetricKey = strings.TrimSpace(query.MetricKey)
query.DateFrom = strings.TrimSpace(query.DateFrom) query.DateFrom = strings.TrimSpace(query.DateFrom)
query.DateTo = strings.TrimSpace(query.DateTo) query.DateTo = strings.TrimSpace(query.DateTo)
if query.Limit <= 0 { if query.Limit <= 0 {
@@ -135,18 +127,18 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
func buildMetricSQL(query MetricQuery) (string, []any) { func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query) where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric` sqlText := `SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage`
if len(where) > 0 { if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ") sqlText += " WHERE " + strings.Join(where, " AND ")
} }
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC, metric_key ASC LIMIT ? OFFSET ?" sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset) args = append(args, query.Limit, query.Offset)
return sqlText, args return sqlText, args
} }
func buildMetricCountSQL(query MetricQuery) (string, []any) { func buildMetricCountSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query) where, args := buildMetricWhere(query)
sqlText := `SELECT COUNT(*) FROM vehicle_daily_metric` sqlText := `SELECT COUNT(*) FROM vehicle_daily_mileage`
if len(where) > 0 { if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ") sqlText += " WHERE " + strings.Join(where, " AND ")
} }
@@ -169,9 +161,6 @@ func buildMetricWhere(query MetricQuery) ([]string, []any) {
if query.Protocol != "" { if query.Protocol != "" {
add("protocol = ?", query.Protocol) add("protocol = ?", query.Protocol)
} }
if query.MetricKey != "" {
add("metric_key = ?", query.MetricKey)
}
if query.DateFrom != "" { if query.DateFrom != "" {
add("stat_date >= ?", query.DateFrom) add("stat_date >= ?", query.DateFrom)
} }
@@ -239,7 +228,6 @@ func parseMetricQuery(r *http.Request) (MetricQuery, error) {
VehicleKey: values.Get("vehicleKey"), VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"), VIN: values.Get("vin"),
Protocol: values.Get("protocol"), Protocol: values.Get("protocol"),
MetricKey: values.Get("metricKey"),
DateFrom: values.Get("dateFrom"), DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"), DateTo: values.Get("dateTo"),
Limit: limit, Limit: limit,

View File

@@ -18,15 +18,15 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0). WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "created_at", "updated_at",
}).AddRow( }).AddRow(
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", MetricDailyTotalMileageKM, 12345.6, "km", "LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 12.3,
12345.6, 12345.6, 13, "TOTAL_MILEAGE_DIFF", time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)), 12345.6, 12357.9, 13, time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
)) ))
repository := NewMetricRepository(db) repository := NewMetricRepository(db)
@@ -43,7 +43,7 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
if len(rows) != 1 { if len(rows) != 1 {
t.Fatalf("row count = %d", len(rows)) t.Fatalf("row count = %d", len(rows))
} }
if rows[0].MetricKey != MetricDailyTotalMileageKM || rows[0].MetricValue != 12345.6 { if rows[0].DailyMileageKM != 12.3 {
t.Fatalf("unexpected row: %#v", rows[0]) t.Fatalf("unexpected row: %#v", rows[0])
} }
if rows[0].VehicleKey != "LKLG7C4E3NA774736" { if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
@@ -63,18 +63,18 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01"). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "created_at", "updated_at",
}).AddRow( }).AddRow(
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", MetricDailyMileageKM, 0.0, "km", "LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", 0.0,
53490.9, 53490.9, 3, "TOTAL_MILEAGE_DIFF", "2026-07-01 22:07:58", "2026-07-01 22:28:25", 53490.9, 53490.9, 3, "2026-07-01 22:07:58", "2026-07-01 22:28:25",
)) ))
handler := NewMetricHandler(NewMetricRepository(db)) handler := NewMetricHandler(NewMetricRepository(db))
@@ -87,7 +87,7 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
} }
body := response.Body.String() body := response.Body.String()
for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"metric_key":"daily_mileage_km"`, `"total":42`} { for _, want := range []string{`"vin":"LB9A32A21R0LS1707"`, `"daily_mileage_km":0`, `"total":42`} {
if !strings.Contains(body, want) { if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
@@ -103,18 +103,18 @@ func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("JT808:013307811254", "JT808"). WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("JT808:013307811254", "JT808", 50, 0). WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "created_at", "updated_at",
}).AddRow( }).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", MetricDailyTotalMileageKM, 10985.7, "km", "JT808:013307811254", "", "2026-07-02", "JT808", 0.0,
10985.7, 10985.7, 1, "TOTAL_MILEAGE_DIFF", "2026-07-02 00:03:00", "2026-07-02 00:03:00", 10985.7, 10985.7, 1, "2026-07-02 00:03:00", "2026-07-02 00:03:00",
)) ))
handler := NewMetricHandler(NewMetricRepository(db)) handler := NewMetricHandler(NewMetricRepository(db))
@@ -127,7 +127,7 @@ func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
} }
body := response.Body.String() body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"metric_value":10985.7`} { for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"daily_mileage_km":0`} {
if !strings.Contains(body, want) { if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
@@ -143,14 +143,14 @@ func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT"). WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, metric_key, metric_value, metric_unit, first_total_mileage_km, latest_total_mileage_km, sample_count, calculation_method, created_at, updated_at FROM vehicle_daily_metric"). mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT", 50, 0). WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "metric_key", "metric_value", "metric_unit", "vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "calculation_method", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "created_at", "updated_at",
})) }))

View File

@@ -1,22 +1,19 @@
package stats package stats
const DailyMetricTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_metric ( const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
id BIGINT PRIMARY KEY AUTO_INCREMENT, id BIGINT PRIMARY KEY AUTO_INCREMENT,
vehicle_key VARCHAR(96) NOT NULL, vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '', vin VARCHAR(32) NOT NULL DEFAULT '',
stat_date DATE NOT NULL, stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
metric_key VARCHAR(64) NOT NULL, daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
metric_value DECIMAL(18,3) NOT NULL,
metric_unit VARCHAR(16) NOT NULL,
first_total_mileage_km DECIMAL(18,3) NULL, first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL, latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0, sample_count BIGINT NOT NULL DEFAULT 0,
calculation_method VARCHAR(64) NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_metric_vehicle (vehicle_key, stat_date, protocol, metric_key), UNIQUE KEY uk_daily_mileage_vehicle (vehicle_key, stat_date, protocol),
KEY idx_vin (vin), KEY idx_vin (vin),
KEY idx_stat_date (stat_date), KEY idx_stat_date (stat_date),
KEY idx_protocol_metric (protocol, metric_key) KEY idx_protocol_date (protocol, stat_date)
)` )`