refactor(go): simplify realtime mysql current tables

This commit is contained in:
lingniu
2026-07-03 07:11:40 +08:00
parent 7c41b81654
commit e89c9fd98f
7 changed files with 31 additions and 26 deletions

View File

@@ -69,6 +69,8 @@ NATS 部署在 Kafka ECS 内网 `172.17.111.56:4222`。
| `vehicle_realtime_location` | `protocol``vin``plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 | | `vehicle_realtime_location` | `protocol``vin``plate`、经纬度、速度、总里程、SOC、事件时间 | Realtime API/projector | 每协议每 VIN 最新位置业务缓存 |
| `vehicle_daily_mileage` | `vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程;无 VIN 数据不进入正式统计表 | | `vehicle_daily_mileage` | `vin``stat_date``protocol``daily_mileage_km`、首末总里程、样本数 | Stat writer | 基于总里程差值的每日里程;无 VIN 数据不进入正式统计表 |
两张实时当前态表均使用 `(protocol, vin)` 作为业务主键,不保留自增 `id``created_at`;最新更新时间使用 `updated_at`
不应重新出现: 不应重新出现:
- `vehicle_daily_metric` - `vehicle_daily_metric`

View File

@@ -50,7 +50,12 @@
- 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。 - 无 VIN 的位置帧仍保留在 `raw_frames.parsed_json`,待 identity/binding 修复后可从 raw 重放补写。
- 生产ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol``vin` 两个 tag。 - 生产ECS TDengine `vehicle_locations` 已在上线前重建为 `protocol``vin` 两个 tag。
5. identity 层只保留两张表 5. MySQL realtime 当前态表不保留代理主键和创建时间
- 原因:`vehicle_realtime_snapshot``vehicle_realtime_location` 都是每协议每 VIN 一行的当前态投影,业务主键就是 `(protocol, vin)`
- 状态schema 使用 `PRIMARY KEY (protocol, vin)`,不保留自增 `id``created_at`,对外只暴露最新 `updated_at`
- 生产:项目上线前可直接重建这两张 MySQL 表,实时数据会从 Kafka 新消息继续投影。
6. identity 层只保留两张表。
- `vehicle_identity_binding`VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。 - `vehicle_identity_binding`VIN 与 plate/phone/device_id 的映射,供 808 等协议反查 VIN。
- `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。 - `jt808_registration`808 注册、鉴权、最新活跃和 VIN 匹配状态。
- 生产RDS 已在上线前删除旧 `vehicle_identity_binding_registration``vehicle_identity_bindings`gateway 启动会自动创建最小 schema。 - 生产RDS 已在上线前删除旧 `vehicle_identity_binding_registration``vehicle_identity_bindings`gateway 启动会自动创建最小 schema。

View File

@@ -30,7 +30,6 @@ type SnapshotRow struct {
EventTime string `json:"event_time,omitempty"` EventTime string `json:"event_time,omitempty"`
ReceivedAt string `json:"received_at,omitempty"` ReceivedAt string `json:"received_at,omitempty"`
EventID string `json:"event_id"` EventID string `json:"event_id"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
} }
@@ -50,7 +49,6 @@ type LocationRow struct {
StatusFlag *int64 `json:"status_flag,omitempty"` StatusFlag *int64 `json:"status_flag,omitempty"`
ReceivedAt string `json:"received_at,omitempty"` ReceivedAt string `json:"received_at,omitempty"`
EventID string `json:"event_id"` EventID string `json:"event_id"`
CreatedAt string `json:"created_at"`
UpdatedAt string `json:"updated_at"` UpdatedAt string `json:"updated_at"`
} }
@@ -74,7 +72,7 @@ func (r *SnapshotQueryRepository) Query(ctx context.Context, query RealtimeTable
query = normalizeRealtimeTableQuery(query) query = normalizeRealtimeTableQuery(query)
sqlText, args := buildRealtimeSelectSQL( sqlText, args := buildRealtimeSelectSQL(
"vehicle_realtime_snapshot", "vehicle_realtime_snapshot",
"protocol, vin, plate, event_time, received_at, event_id, created_at, updated_at", "protocol, vin, plate, event_time, received_at, event_id, updated_at",
query, query,
) )
rows, err := r.db.QueryContext(ctx, sqlText, args...) rows, err := r.db.QueryContext(ctx, sqlText, args...)
@@ -86,13 +84,12 @@ func (r *SnapshotQueryRepository) Query(ctx context.Context, query RealtimeTable
out := make([]SnapshotRow, 0) out := make([]SnapshotRow, 0)
for rows.Next() { for rows.Next() {
var row SnapshotRow var row SnapshotRow
var eventTime, receivedAt, createdAt, updatedAt scanSQLDateTime var eventTime, receivedAt, updatedAt scanSQLDateTime
if err := rows.Scan(&row.Protocol, &row.VIN, &row.Plate, &eventTime, &receivedAt, &row.EventID, &createdAt, &updatedAt); err != nil { if err := rows.Scan(&row.Protocol, &row.VIN, &row.Plate, &eventTime, &receivedAt, &row.EventID, &updatedAt); err != nil {
return nil, err return nil, err
} }
row.EventTime = eventTime.String row.EventTime = eventTime.String
row.ReceivedAt = receivedAt.String row.ReceivedAt = receivedAt.String
row.CreatedAt = createdAt.String
row.UpdatedAt = updatedAt.String row.UpdatedAt = updatedAt.String
out = append(out, row) out = append(out, row)
} }
@@ -119,7 +116,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
query = normalizeRealtimeTableQuery(query) query = normalizeRealtimeTableQuery(query)
sqlText, args := buildRealtimeSelectSQL( sqlText, args := buildRealtimeSelectSQL(
"vehicle_realtime_location", "vehicle_realtime_location",
"protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, created_at, updated_at", "protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at",
query, query,
) )
rows, err := r.db.QueryContext(ctx, sqlText, args...) rows, err := r.db.QueryContext(ctx, sqlText, args...)
@@ -131,7 +128,7 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
out := make([]LocationRow, 0) out := make([]LocationRow, 0)
for rows.Next() { for rows.Next() {
var row LocationRow var row LocationRow
var eventTime, receivedAt, createdAt, updatedAt scanSQLDateTime var eventTime, receivedAt, updatedAt scanSQLDateTime
var speed, mileage, soc, altitude, direction sql.NullFloat64 var speed, mileage, soc, altitude, direction sql.NullFloat64
var alarm, status sql.NullInt64 var alarm, status sql.NullInt64
if err := rows.Scan( if err := rows.Scan(
@@ -150,7 +147,6 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
&status, &status,
&receivedAt, &receivedAt,
&row.EventID, &row.EventID,
&createdAt,
&updatedAt, &updatedAt,
); err != nil { ); err != nil {
return nil, err return nil, err
@@ -164,7 +160,6 @@ func (r *LocationQueryRepository) Query(ctx context.Context, query RealtimeTable
row.AlarmFlag = nullableInt(alarm) row.AlarmFlag = nullableInt(alarm)
row.StatusFlag = nullableInt(status) row.StatusFlag = nullableInt(status)
row.ReceivedAt = receivedAt.String row.ReceivedAt = receivedAt.String
row.CreatedAt = createdAt.String
row.UpdatedAt = updatedAt.String row.UpdatedAt = updatedAt.String
out = append(out, row) out = append(out, row)
} }

View File

@@ -20,12 +20,12 @@ func TestSnapshotQueryHandlerReturnsRealtimeSnapshots(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_snapshot"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_snapshot").
WithArgs("GB32960", "LB9A32A21R0LS1707"). WithArgs("GB32960", "LB9A32A21R0LS1707").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1))
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, received_at, event_id, created_at, updated_at FROM vehicle_realtime_snapshot"). mock.ExpectQuery("SELECT protocol, vin, plate, event_time, received_at, event_id, updated_at FROM vehicle_realtime_snapshot").
WithArgs("GB32960", "LB9A32A21R0LS1707", 20, 0). WithArgs("GB32960", "LB9A32A21R0LS1707", 20, 0).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"protocol", "vin", "plate", "event_time", "received_at", "event_id", "created_at", "updated_at", "protocol", "vin", "plate", "event_time", "received_at", "event_id", "updated_at",
}).AddRow( }).AddRow(
"GB32960", "LB9A32A21R0LS1707", "浙A12345", "2026-07-02 16:01:02.123", "2026-07-02 16:01:03.456", "evt-1", "2026-07-02 16:01:03", "2026-07-02 16:01:04", "GB32960", "LB9A32A21R0LS1707", "浙A12345", "2026-07-02 16:01:02.123", "2026-07-02 16:01:03.456", "evt-1", "2026-07-02 16:01:04",
)) ))
handler := NewSnapshotQueryHandler(NewSnapshotQueryRepository(db)) handler := NewSnapshotQueryHandler(NewSnapshotQueryRepository(db))
@@ -43,6 +43,9 @@ func TestSnapshotQueryHandlerReturnsRealtimeSnapshots(t *testing.T) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
} }
if strings.Contains(body, "created_at") {
t.Fatalf("snapshot response should not expose created_at: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil { if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err) t.Fatalf("sql expectations: %v", err)
} }
@@ -58,14 +61,14 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) {
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_location"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM vehicle_realtime_location").
WithArgs("JT808", "粤B98765"). WithArgs("JT808", "粤B98765").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(1))
mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, created_at, updated_at FROM vehicle_realtime_location"). mock.ExpectQuery("SELECT protocol, vin, plate, event_time, latitude, longitude, speed_kmh, total_mileage_km, soc_percent, altitude_m, direction_deg, alarm_flag, status_flag, received_at, event_id, updated_at FROM vehicle_realtime_location").
WithArgs("JT808", "粤B98765", 10, 10). WithArgs("JT808", "粤B98765", 10, 10).
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km", "protocol", "vin", "plate", "event_time", "latitude", "longitude", "speed_kmh", "total_mileage_km",
"soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "created_at", "updated_at", "soc_percent", "altitude_m", "direction_deg", "alarm_flag", "status_flag", "received_at", "event_id", "updated_at",
}).AddRow( }).AddRow(
"JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9, "JT808", "LKLG7C4E3NA774736", "粤B98765", "2026-07-02 16:11:02.000", 30.123456, 120.654321, 54.3, 48798.9,
nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:03", "2026-07-02 16:11:04", nil, 19.0, 88.0, int64(0), int64(3), "2026-07-02 16:11:03.000", "evt-2", "2026-07-02 16:11:04",
)) ))
handler := NewLocationQueryHandler(NewLocationQueryRepository(db)) handler := NewLocationQueryHandler(NewLocationQueryRepository(db))
@@ -83,6 +86,9 @@ func TestLocationQueryHandlerReturnsRealtimeLocations(t *testing.T) {
t.Fatalf("response missing %s: %s", want, body) t.Fatalf("response missing %s: %s", want, body)
} }
} }
if strings.Contains(body, "created_at") {
t.Fatalf("location response should not expose created_at: %s", body)
}
if err := mock.ExpectationsWereMet(); err != nil { if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err) t.Fatalf("sql expectations: %v", err)
} }

View File

@@ -93,7 +93,6 @@ func realtimeOpenAPISpec() map[string]any {
"event_time": stringSchema("2026-07-02 16:01:02.123"), "event_time": stringSchema("2026-07-02 16:01:02.123"),
"received_at": stringSchema("2026-07-02 16:01:03.456"), "received_at": stringSchema("2026-07-02 16:01:03.456"),
"event_id": stringSchema("event id"), "event_id": stringSchema("event id"),
"created_at": stringSchema("2026-07-02 16:01:03"),
"updated_at": stringSchema("2026-07-02 16:01:04"), "updated_at": stringSchema("2026-07-02 16:01:04"),
}, },
}, },
@@ -115,7 +114,6 @@ func realtimeOpenAPISpec() map[string]any {
"status_flag": integerSchema(3), "status_flag": integerSchema(3),
"received_at": stringSchema("2026-07-02 16:11:03.000"), "received_at": stringSchema("2026-07-02 16:11:03.000"),
"event_id": stringSchema("event id"), "event_id": stringSchema("event id"),
"created_at": stringSchema("2026-07-02 16:11:03"),
"updated_at": stringSchema("2026-07-02 16:11:04"), "updated_at": stringSchema("2026-07-02 16:11:04"),
}, },
}, },

View File

@@ -264,16 +264,14 @@ func hasRealtimePayload(env envelope.FrameEnvelope) bool {
} }
const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot ( const realtimeSnapshotTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_snapshot (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '', vin VARCHAR(32) NOT NULL DEFAULT '',
plate VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '',
event_time DATETIME(3) NULL, event_time DATETIME(3) NULL,
received_at DATETIME(3) NULL, received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '', event_id VARCHAR(64) NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_realtime_snapshot_vin (protocol, vin), PRIMARY KEY (protocol, vin),
KEY idx_vin (vin), KEY idx_vin (vin),
KEY idx_protocol_updated (protocol, updated_at) KEY idx_protocol_updated (protocol, updated_at)
)` )`
@@ -291,7 +289,6 @@ ON DUPLICATE KEY UPDATE
` `
const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location ( const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_location (
id BIGINT PRIMARY KEY AUTO_INCREMENT,
protocol VARCHAR(32) NOT NULL, protocol VARCHAR(32) NOT NULL,
vin VARCHAR(32) NOT NULL DEFAULT '', vin VARCHAR(32) NOT NULL DEFAULT '',
plate VARCHAR(32) NOT NULL DEFAULT '', plate VARCHAR(32) NOT NULL DEFAULT '',
@@ -307,9 +304,8 @@ const realtimeLocationTableSQL = `CREATE TABLE IF NOT EXISTS vehicle_realtime_lo
status_flag BIGINT NULL, status_flag BIGINT NULL,
received_at DATETIME(3) NULL, received_at DATETIME(3) NULL,
event_id VARCHAR(64) NOT NULL DEFAULT '', event_id VARCHAR(64) NOT NULL DEFAULT '',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
UNIQUE KEY uk_realtime_location_vin (protocol, vin), PRIMARY KEY (protocol, vin),
KEY idx_vin (vin), KEY idx_vin (vin),
KEY idx_protocol_updated (protocol, updated_at), KEY idx_protocol_updated (protocol, updated_at),
KEY idx_location (longitude, latitude) KEY idx_location (longitude, latitude)

View File

@@ -52,11 +52,14 @@ func TestSnapshotWriterEnsuresSchemaAndUpsertsCoreSnapshot(t *testing.T) {
if strings.Contains(call.query, "parsed_json") || strings.Contains(call.query, "fields_json") { if strings.Contains(call.query, "parsed_json") || strings.Contains(call.query, "fields_json") {
t.Fatalf("realtime schema should not contain json payload columns: %s", call.query) t.Fatalf("realtime schema should not contain json payload columns: %s", call.query)
} }
for _, column := range []string{"message_id", "sequence_id", "source_endpoint"} { for _, column := range []string{"id BIGINT", "created_at", "message_id", "sequence_id", "source_endpoint"} {
if strings.Contains(call.query, column) { if strings.Contains(call.query, column) {
t.Fatalf("realtime schema should not contain %s: %s", column, call.query) t.Fatalf("realtime schema should not contain %s: %s", column, call.query)
} }
} }
if !strings.Contains(call.query, "PRIMARY KEY (protocol, vin)") {
t.Fatalf("realtime current-state table should key by protocol/vin: %s", call.query)
}
} }
upsert := exec.calls[2] upsert := exec.calls[2]
if !strings.Contains(upsert.query, "ON DUPLICATE KEY UPDATE") { if !strings.Contains(upsert.query, "ON DUPLICATE KEY UPDATE") {