refactor(go): simplify daily mileage table key

This commit is contained in:
lingniu
2026-07-03 07:16:49 +08:00
parent e89c9fd98f
commit 04bafb4cd6
6 changed files with 24 additions and 19 deletions

View File

@@ -69,6 +69,8 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。
| `vehicle_realtime_location` | `protocol``vin``plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 | | `vehicle_realtime_location` | `protocol``vin``plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 |
| `vehicle_daily_mileage` | `vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程;无 VIN 数据不进入正式统计表 | | `vehicle_daily_mileage` | `vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程;无 VIN 数据不进入正式统计表 |
日里程表使用 `(vin, stat_date, protocol)` 作为业务主键,不保留自增 `id``created_at`;首末总里程和样本数保留用于解释差值计算。
两张实时当前态表均使用 `(protocol, vin)` 作为业务主键,不保留自增 `id``created_at`;最新更新时间使用 `updated_at` 两张实时当前态表均使用 `(protocol, vin)` 作为业务主键,不保留自增 `id``created_at`;最新更新时间使用 `updated_at`
不应重新出现: 不应重新出现:

View File

@@ -23,7 +23,7 @@
| TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息、phone/device_id/vehicle_key 兜底标识 | | TDengine | `vehicle_locations` | 时间、VIN/协议标签、经纬度、速度、方向、SOC、总里程等位置核心字段 | 完整协议 JSON、注册鉴权信息、phone/device_id/vehicle_key 兜底标识 |
| MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON | | MySQL | `vehicle_realtime_snapshot` | 每个协议+VIN 的最新事件时间、接收时间、车牌、事件 ID | phone、device、source endpoint、完整 JSON |
| MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 | | MySQL | `vehicle_realtime_location` | 每个协议+VIN 的最新位置核心字段 | raw、parsed JSON、消息头内部字段 |
| MySQL | `vehicle_daily_mileage` | 日期、VIN、协议、日里程、首末总里程、样本数 | 临时 vehicle key、泛化 metric key/value、每帧细节、位置点列表 | | MySQL | `vehicle_daily_mileage` | 日期、VIN、协议、日里程、首末总里程、样本数 | 临时 vehicle key、泛化 metric key/value、自增 id、created_at、每帧细节、位置点列表 |
| MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、device_id 映射 | 注册历史、协议状态 | | MySQL | `vehicle_identity_binding` | 人工维护或导入的 VIN、车牌、phone、device_id 映射 | 注册历史、协议状态 |
| MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态、来源端点 | GB32960/MQTT 注册信息、位置历史 | | MySQL | `jt808_registration` | JT808 phone 主键下的注册、鉴权、VIN 匹配状态、来源端点 | GB32960/MQTT 注册信息、位置历史 |
| Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议最新完整 parsed 状态 | 历史数据、统计结果 | | Redis | `vehicle:realtime-raw:{protocol}:{vin}` | 每协议最新完整 parsed 状态 | 历史数据、统计结果 |
@@ -43,6 +43,7 @@
3. `vehicle_daily_mileage` 不再接收临时 `vehicle_key` 3. `vehicle_daily_mileage` 不再接收临时 `vehicle_key`
- 原因:日统计是正式业务指标,应只统计已经定位到 VIN 的车辆。 - 原因:日统计是正式业务指标,应只统计已经定位到 VIN 的车辆。
- 无 VIN 的 JT808 等数据保留在 raw/Redis/session 中用于排查和待绑定,不进入正式车辆指标。 - 无 VIN 的 JT808 等数据保留在 raw/Redis/session 中用于排查和待绑定,不进入正式车辆指标。
- 状态schema 使用 `PRIMARY KEY (vin, stat_date, protocol)`,不保留自增 `id``created_at`;首末总里程和样本数保留为日里程计算状态。
- 生产RDS 已在上线前删除泛化 `vehicle_daily_metric`,改为专用 `vehicle_daily_mileage` - 生产RDS 已在上线前删除泛化 `vehicle_daily_metric`,改为专用 `vehicle_daily_mileage`
4. `vehicle_locations` 不再接收临时身份兜底字段。 4. `vehicle_locations` 不再接收临时身份兜底字段。

View File

@@ -121,8 +121,13 @@ func TestWriterEnsuresSchemaAndUpsertsDailyMileage(t *testing.T) {
if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_mileage") { if !strings.Contains(exec.calls[0].query, "CREATE TABLE IF NOT EXISTS vehicle_daily_mileage") {
t.Fatalf("unexpected schema sql: %s", exec.calls[0].query) t.Fatalf("unexpected schema sql: %s", exec.calls[0].query)
} }
if strings.Contains(exec.calls[0].query, "vehicle_key") { for _, column := range []string{"vehicle_key", "id BIGINT", "AUTO_INCREMENT", "created_at"} {
t.Fatalf("schema should not include vehicle_key: %s", exec.calls[0].query) if strings.Contains(exec.calls[0].query, column) {
t.Fatalf("schema should not include %s: %s", column, exec.calls[0].query)
}
}
if !strings.Contains(exec.calls[0].query, "PRIMARY KEY (vin, stat_date, protocol)") {
t.Fatalf("daily mileage table should key by vin/stat_date/protocol: %s", exec.calls[0].query)
} }
if len(exec.calls) != 2 { if len(exec.calls) != 2 {
t.Fatalf("exec calls = %d", len(exec.calls)) t.Fatalf("exec calls = %d", len(exec.calls))

View File

@@ -33,7 +33,6 @@ type MetricRow struct {
FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"` FirstTotalMileageKM *float64 `json:"first_total_mileage_km,omitempty"`
LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"` LatestTotalMileageKM *float64 `json:"latest_total_mileage_km,omitempty"`
SampleCount int64 `json:"sample_count"` SampleCount int64 `json:"sample_count"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
} }
@@ -61,7 +60,6 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
for rows.Next() { for rows.Next() {
var row MetricRow var row MetricRow
var statDate scanDate var statDate scanDate
var createdAt scanDateTime
var updatedAt scanDateTime var updatedAt scanDateTime
var first sql.NullFloat64 var first sql.NullFloat64
var latest sql.NullFloat64 var latest sql.NullFloat64
@@ -73,13 +71,11 @@ func (r *MetricRepository) Query(ctx context.Context, query MetricQuery) ([]Metr
&first, &first,
&latest, &latest,
&row.SampleCount, &row.SampleCount,
&createdAt,
&updatedAt, &updatedAt,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
row.StatDate = statDate.String row.StatDate = statDate.String
row.CreatedAt = createdAt.String
row.UpdatedAt = updatedAt.String row.UpdatedAt = updatedAt.String
if first.Valid { if first.Valid {
row.FirstTotalMileageKM = &first.Float64 row.FirstTotalMileageKM = &first.Float64
@@ -123,7 +119,7 @@ func normalizeMetricQuery(query MetricQuery) MetricQuery {
func buildMetricSQL(query MetricQuery) (string, []any) { func buildMetricSQL(query MetricQuery) (string, []any) {
where, args := buildMetricWhere(query) where, args := buildMetricWhere(query)
sqlText := `SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage` sqlText := `SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, updated_at FROM vehicle_daily_mileage`
if len(where) > 0 { if len(where) > 0 {
sqlText += " WHERE " + strings.Join(where, " AND ") sqlText += " WHERE " + strings.Join(where, " AND ")
} }

View File

@@ -18,15 +18,15 @@ func TestMetricRepositoryQueriesDailyMetricsWithFilters(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage"). mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, updated_at FROM vehicle_daily_mileage").
WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0). WithArgs("LKLG7C4E3NA774736", "JT808", "2026-07-01", "2026-07-01", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "daily_mileage_km", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "updated_at",
}).AddRow( }).AddRow(
"LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 12.3, "LKLG7C4E3NA774736", time.Date(2026, 7, 1, 0, 0, 0, 0, time.FixedZone("Asia/Shanghai", 8*3600)), "JT808", 12.3,
12345.6, 12357.9, 13, time.Date(2026, 7, 1, 22, 49, 11, 0, time.FixedZone("Asia/Shanghai", 8*3600)), time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)), 12345.6, 12357.9, 13, time.Date(2026, 7, 1, 23, 9, 36, 0, time.FixedZone("Asia/Shanghai", 8*3600)),
)) ))
repository := NewMetricRepository(db) repository := NewMetricRepository(db)
@@ -63,15 +63,15 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01"). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(42))
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage"). mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, updated_at FROM vehicle_daily_mileage").
WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0). WithArgs("LB9A32A21R0LS1707", "GB32960", "2020-07-01", "2020-07-01", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "daily_mileage_km", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "updated_at",
}).AddRow( }).AddRow(
"LB9A32A21R0LS1707", "2020-07-01", "GB32960", 0.0, "LB9A32A21R0LS1707", "2020-07-01", "GB32960", 0.0,
53490.9, 53490.9, 3, "2026-07-01 22:07:58", "2026-07-01 22:28:25", 53490.9, 53490.9, 3, "2026-07-01 22:28:25",
)) ))
handler := NewMetricHandler(NewMetricRepository(db)) handler := NewMetricHandler(NewMetricRepository(db))
@@ -89,6 +89,9 @@ func TestMetricHandlerReturnsDailyMetrics(t *testing.T) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
} }
if strings.Contains(body, "created_at") {
t.Fatalf("daily mileage response should not expose created_at: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil { if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err) t.Fatalf("sql expectations: %v", err)
} }
@@ -103,12 +106,12 @@ func TestMetricHandlerReturnsEmptyItemsArrayWhenNoRows(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT"). WithArgs("YUTONG_MQTT").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(0))
mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, created_at, updated_at FROM vehicle_daily_mileage"). mock.ExpectQuery("SELECT vin, stat_date, protocol, daily_mileage_km, first_total_mileage_km, latest_total_mileage_km, sample_count, updated_at FROM vehicle_daily_mileage").
WithArgs("YUTONG_MQTT", 50, 0). WithArgs("YUTONG_MQTT", 50, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"vin", "stat_date", "protocol", "daily_mileage_km", "vin", "stat_date", "protocol", "daily_mileage_km",
"first_total_mileage_km", "latest_total_mileage_km", "sample_count", "first_total_mileage_km", "latest_total_mileage_km", "sample_count",
"created_at", "updated_at", "updated_at",
})) }))
handler := NewMetricHandler(NewMetricRepository(db)) handler := NewMetricHandler(NewMetricRepository(db))

View File

@@ -1,7 +1,6 @@
package stats package stats
const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage ( const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
vin VARCHAR(32) NOT NULL, vin VARCHAR(32) NOT NULL,
stat_date DATE NOT NULL, stat_date DATE NOT NULL,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
@@ -9,9 +8,8 @@ const DailyMileageTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_daily_mileage (
first_total_mileage_km DECIMAL(18,3) NULL, first_total_mileage_km DECIMAL(18,3) NULL,
latest_total_mileage_km DECIMAL(18,3) NULL, latest_total_mileage_km DECIMAL(18,3) NULL,
sample_count BIGINT NOT NULL DEFAULT 0, sample_count BIGINT NOT NULL DEFAULT 0,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_daily_mileage_vin (vin, stat_date, protocol), PRIMARY KEY (vin, stat_date, protocol),
KEY idx_vin (vin), KEY idx_vin (vin),
KEY idx_stat_date (stat_date), KEY idx_stat_date (stat_date),
KEY idx_protocol_date (protocol, stat_date) KEY idx_protocol_date (protocol, stat_date)