refactor(go): make daily mileage vin only

This commit is contained in:
lingniu
2026-07-02 21:02:02 +08:00
parent b23ae3410c
commit bec6326103
8 changed files with 43 additions and 106 deletions

View File

@@ -49,7 +49,7 @@ flowchart LR
| TDengine | `vehicle_locations` | High-volume location history | time, vin, protocol, longitude, latitude, speed, direction, mileage | full parsed JSON |
| MySQL | `vehicle_realtime_snapshot` | Latest per-protocol vehicle heartbeat | protocol, vin, plate, event time, received time | phone, device, message sequence, parsed JSON |
| MySQL | `vehicle_realtime_location` | Latest business location cache | vin, plate, location, speed, mileage, SOC, event time | full raw payload and protocol internals |
| MySQL | `vehicle_daily_mileage` | Queryable daily mileage | vin or vehicle key, date, protocol, daily mileage, source mileage | generic metric key/value rows, per-frame raw details |
| MySQL | `vehicle_daily_mileage` | Queryable daily mileage | vin, date, protocol, daily mileage, source mileage | temporary vehicle keys, generic metric key/value rows, per-frame raw details |
| MySQL | `vehicle_identity_binding` | Manual identity mapping | vin, plate, phone, device id | registration history |
| MySQL | `jt808_registration` | JT808 registration and auth trace | phone, device id, plate, auth code, vin match state, first/latest seen | GB32960 or MQTT records |
| Redis | `vehicle:latest:{vehicleKey}` | Latest merged realtime state | cross-protocol latest fields | historical data |

View File

@@ -67,7 +67,7 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。
| `jt808_registration` | `phone``device_id``plate``vin``manufacturer``auth_token``source_endpoint`、首次/最新注册鉴权时间 | Gateway | JT808 注册、鉴权和 VIN 匹配状态 |
| `vehicle_realtime_snapshot` | `protocol``vin``plate``event_time``received_at``event_id` | Realtime API/projector | 每协议每 VIN 最新事件轻量快照 |
| `vehicle_realtime_location` | `protocol``vin``plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 |
| `vehicle_daily_mileage` | `vehicle_key``vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程 |
| `vehicle_daily_mileage` | `vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程;无 VIN 数据不进入正式统计表 |
不应重新出现:

View File

@@ -23,7 +23,7 @@
| TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息 |
| MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON |
| MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 |
| MySQL | `vehicle_daily_mileage` | 日期、VIN/临时 vehicle key、协议、日里程、首末总里程、样本数 | 泛化 metric key/value、每帧细节、位置点列表 |
| MySQL | `vehicle_daily_mileage` | 日期、VIN、协议、日里程、首末总里程、样本数 | 临时 vehicle key、泛化 metric key/value、每帧细节、位置点列表 |
| MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、device_id 映射 | 注册历史、协议状态 |
| MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态、来源端点 | GB32960/MQTT 注册信息、位置历史 |
| Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议最新完整 parsed 状态 | 历史数据、统计结果 |
@@ -40,9 +40,9 @@
- 兼容raw 查询只选择公共列,不依赖 `fields_json`;旧表保留该列也不影响查询。
- 生产ECS TDengine 历史库已在上线前重建,`raw_frames` 不再包含 `fields_json`
3. `vehicle_daily_mileage` `vehicle_key``vin` 同时存在
- 当前保留原因JT808 可能先用 phone 作为临时 key后续才绑定 VIN
- 收敛方向:业务查询以 VIN 为主;没有 VIN 的数据只用于排查和待绑定,不作为正式车辆指标。
3. `vehicle_daily_mileage` 不再接收临时 `vehicle_key`
- 原因:日统计是正式业务指标,应只统计已经定位到 VIN 的车辆
- 无 VIN 的 JT808 等数据保留在 raw/Redis/session 中用于排查和待绑定,不进入正式车辆指标。
- 生产RDS 已在上线前删除泛化 `vehicle_daily_metric`,改为专用 `vehicle_daily_mileage`
4. identity 层只保留两张表。

View File

@@ -21,7 +21,6 @@ type Writer struct {
}
type MetricSample struct {
VehicleKey string
VIN string
Protocol envelope.Protocol
StatDate string
@@ -52,7 +51,6 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
}
for _, sample := range samples {
if _, err := w.exec.ExecContext(ctx, upsertDailyMileageSQL,
sample.VehicleKey,
sample.VIN,
sample.StatDate,
string(sample.Protocol),
@@ -66,8 +64,7 @@ func (w *Writer) Append(ctx context.Context, env envelope.FrameEnvelope) error {
func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]MetricSample, error) {
vin := strings.TrimSpace(env.VIN)
vehicleKey := strings.TrimSpace(env.VehicleKey())
if vehicleKey == "" || strings.HasSuffix(vehicleKey, ":unknown") {
if vin == "" {
return nil, nil
}
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
@@ -89,7 +86,6 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
}
statDate := time.UnixMilli(eventMS).In(loc).Format("2006-01-02")
return []MetricSample{{
VehicleKey: vehicleKey,
VIN: vin,
Protocol: env.Protocol,
StatDate: statDate,
@@ -99,11 +95,10 @@ func SamplesFromEnvelope(env envelope.FrameEnvelope, loc *time.Location) ([]Metr
const upsertDailyMileageSQL = `
INSERT INTO vehicle_daily_mileage
(vehicle_key, vin, stat_date, protocol, daily_mileage_km,
(vin, stat_date, protocol, daily_mileage_km,
first_total_mileage_km, latest_total_mileage_km, sample_count)
VALUES (?, ?, ?, ?, 0, ?, ?, 1)
VALUES (?, ?, ?, 0, ?, ?, 1)
ON DUPLICATE KEY UPDATE
vin = IF(VALUES(vin) <> '', VALUES(vin), vin),
first_total_mileage_km = CASE
WHEN first_total_mileage_km IS NULL OR first_total_mileage_km <= 0
THEN VALUES(first_total_mileage_km)

View File

@@ -28,8 +28,8 @@ func TestSamplesFromEnvelopeDerivesDailyMileageSample(t *testing.T) {
if len(samples) != 1 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "LNBVIN00000000001" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
if samples[0].VIN != "LNBVIN00000000001" {
t.Fatalf("vin = %q", samples[0].VIN)
}
if samples[0].TotalMileageKM != 10241.2 {
t.Fatalf("total mileage = %v", samples[0].TotalMileageKM)
@@ -39,7 +39,7 @@ func TestSamplesFromEnvelopeDerivesDailyMileageSample(t *testing.T) {
}
}
func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
func TestSamplesFromEnvelopeSkipsMissingVIN(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
@@ -52,18 +52,12 @@ func TestSamplesFromEnvelopeUsesVehicleKeyWhenVINIsMissing(t *testing.T) {
if err != nil {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 1 {
t.Fatalf("sample count = %d", len(samples))
}
if samples[0].VehicleKey != "JT808:13307811254" {
t.Fatalf("vehicle key = %q", samples[0].VehicleKey)
}
if samples[0].VIN != "" {
t.Fatalf("vin should stay empty for unresolved JT808 identity, got %q", samples[0].VIN)
if len(samples) != 0 {
t.Fatalf("expected no samples without VIN, got %#v", samples)
}
}
func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
func TestSamplesFromEnvelopeSkipsMissingVINOrMileage(t *testing.T) {
samples, err := SamplesFromEnvelope(envelope.FrameEnvelope{
Protocol: envelope.ProtocolJT808,
Fields: map[string]any{envelope.FieldTotalMileageKM: 1.2},
@@ -72,7 +66,7 @@ func TestSamplesFromEnvelopeSkipsMissingVehicleKeyOrMileage(t *testing.T) {
t.Fatalf("SamplesFromEnvelope() error = %v", err)
}
if len(samples) != 0 {
t.Fatalf("expected no samples without vehicle key, got %#v", samples)
t.Fatalf("expected no samples without VIN, got %#v", samples)
}
samples, err = SamplesFromEnvelope(envelope.FrameEnvelope{
@@ -127,8 +121,8 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_mileage") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
}
if !strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should include vehicle_key: %s", exec.calls[0].query)
if strings.Contains(exec.calls[0].query, "vehicle_key") {
t.Fatalf("schema should not include vehicle_key: %s", exec.calls[0].query)
}
if len(exec.calls) != 2 {
t.Fatalf("exec calls = %d", len(exec.calls))
@@ -143,7 +137,7 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
t.Fatalf("upsert should ignore legacy zero first mileage: %s", exec.calls[1].query)
}
if got := exec.calls[1].args[0]; got != "LNBVIN00000000002" {
t.Fatalf("first upsert arg should be vehicle_key, got %#v", got)
t.Fatalf("first upsert arg should be vin, got %#v", got)
}
}

View File

@@ -17,17 +17,15 @@ type Queryer interface {
}
type MetricQuery struct {
VehicleKey string
VIN string
Protocol string
DateFrom string
DateTo string
Limit int
Offset int
VIN string
Protocol string
DateFrom string
DateTo string
Limit int
Offset int
}
type MetricRow struct {
VehicleKey string `json:"vehicle_key"`
VIN string `json:"vin"`
StatDate string `json:"stat_date"`
Protocol string `json:"protocol"`
@@ -68,7 +66,6 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
var first sql.NullFloat64
var latest sql.NullFloat64
if err := rows.Scan(
&row.VehicleKey,
&row.VIN,
&statDate,
&row.Protocol,
@@ -114,7 +111,6 @@ func (r *MetricRepository) Count(ctx context.Context, query MetricQuery) (int64,
}
func normalizeMetricQuery(query MetricQuery) MetricQuery {
query.VehicleKey = strings.TrimSpace(query.VehicleKey)
query.VIN = strings.TrimSpace(query.VIN)
query.Protocol = strings.ToUpper(strings.TrimSpace(query.Protocol))
query.DateFrom = strings.TrimSpace(query.DateFrom)
@@ -127,11 +123,11 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query)
sqlText := `SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage`
sqlText := `SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage`
if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ")
}
sqlText += " ORDER BY stat_date DESC, vehicle_key ASC, protocol ASC LIMIT ? OFFSET ?"
sqlText += " ORDER BY stat_date DESC, vin ASC, protocol ASC LIMIT ? OFFSET ?"
args = append(args, query.Limit, query.Offset)
return sqlText, args
}
@@ -155,9 +151,6 @@ func buildMetricWhere(query MetricQuery) ([]string, []any) {
if query.VIN != "" {
add("vin = ?", query.VIN)
}
if query.VehicleKey != "" {
add("vehicle_key = ?", query.VehicleKey)
}
if query.Protocol != "" {
add("protocol = ?", query.Protocol)
}
@@ -225,13 +218,12 @@ func parseMetricQuery(r *http.Request) (MetricQuery, error) {
return MetricQuery{}, err
}
query := MetricQuery{
VehicleKey: values.Get("vehicleKey"),
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
VIN: values.Get("vin"),
Protocol: values.Get("protocol"),
DateFrom: values.Get("dateFrom"),
DateTo: values.Get("dateTo"),
Limit: limit,
Offset: offset,
}
if !validDate(query.DateFrom) || !validDate(query.DateTo) {
return MetricQuery{}, errors.New("dateFrom/dateTo must use YYYY-MM-DD")

View File

@@ -18,14 +18,14 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at",
}).AddRow(
"LKLG7C4E3NA774736", "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 12.3,
"LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 12.3,
12345.6, 12357.9, 13, time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
))
@@ -46,9 +46,6 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
if rows[0].DailyMileageKM != 12.3 {
t.Fatalf("unexpected row: %#v", rows[0])
}
if rows[0].VehicleKey != "LKLG7C4E3NA774736" {
t.Fatalf("vehicle key = %q", rows[0].VehicleKey)
}
if rows[0].StatDate != "2026-07-01" || rows[0].UpdatedAt != "2026-07-01 23:09:36" {
t.Fatalf("unexpected time formatting: %#v", rows[0])
}
@@ -66,14 +63,14 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at",
}).AddRow(
"LB9A32A21R0LS1707", "LB9A32A21R0LS1707", "2020-07-01", "GB32960", 0.0,
"LB9A32A21R0LS1707", "2020-07-01", "GB32960", 0.0,
53490.9, 53490.9, 3, "2026-07-01 22:07:58", "2026-07-01 22:28:25",
))
@@ -97,46 +94,6 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
}
}
func TestMetricHandlerFiltersByVehicleKey(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
}
defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("JT808:013307811254", "JT808").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(12))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("JT808:013307811254", "JT808", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at",
}).AddRow(
"JT808:013307811254", "", "2026-07-02", "JT808", 0.0,
10985.7, 10985.7, 1, "2026-07-02 00:03:00", "2026-07-02 00:03:00",
))
handler := NewMetricHandler(NewMetricRepository(db))
request := httptest.NewRequest(http.MethodGet, "/api/stats/daily-metrics?vehicleKey=JT808:013307811254&protocol=JT808", nil)
response := httptest.NewRecorder()
handler.ServeHTTP(response, request)
if response.Code != http.StatusOK {
t.Fatalf("status = %d body=%s", response.Code, response.Body.String())
}
body := response.Body.String()
for _, want := range []string{`"vehicle_key":"JT808:013307811254"`, `"vin":""`, `"daily_mileage_km":0`} {
if !strings.Contains(body, want) {
t.Fatalf("response missing %s: %s", want, body)
}
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
}
}
func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
db, mock, err := sqlmock.New()
if err != nil {
@@ -146,10 +103,10 @@ func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vehicle_key, vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{
"vehicle_key", "vin", "stat_date", "protocol", "daily_mileage_km",
"vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at",
}))

View File

@@ -2,8 +2,7 @@ package stats
const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vehicle_key VARCHAR(96) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '',
vin VARCHAR(32) NOT NULL,
stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL,
daily_mileage_km DECIMAL(18,3) NOT NULL DEFAULT 0,
@@ -12,7 +11,7 @@ const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
sample_count BIGINT NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_mileage_vehicle (vehicle_key, stat_date, protocol),
UNIQUE KEY uk_daily_mileage_vin (vin, stat_date, protocol),
KEY idx_vin (vin),
KEY idx_stat_date (stat_date),
KEY idx_protocol_date (protocol, stat_date)