refactor(go): stop duplicate mileage history writes

This commit is contained in:
lingniu
2026-07-02 19:42:53 +08:00
parent be559c0f99
commit 0bbe87e6fc
7 changed files with 30 additions and 73 deletions

View File

@@ -31,8 +31,8 @@
## 当前应收敛的重复点 ## 当前应收敛的重复点
1. `vehicle_mileage_points``vehicle_locations.total_mileage_km` 重复。 1. `vehicle_mileage_points``vehicle_locations.total_mileage_km` 重复。
- 目标:停止写入 `vehicle_mileage_points` - 状态Go 写入链路已停止创建和写入 `vehicle_mileage_points`
- 兼容:`/api/history/mileage-points` 改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL` - 兼容:`/api/history/mileage-points` 改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL`
- 删除:确认线上无直接读表后,再删除 TDengine stable 和子表。 - 删除:确认线上无直接读表后,再删除 TDengine stable 和子表。
2. `raw_frames.fields_json``raw_frames.parsed_json``vehicle_locations` 重复。 2. `raw_frames.fields_json``raw_frames.parsed_json``vehicle_locations` 重复。

View File

@@ -37,7 +37,7 @@
- Modify: `go/vehicle-gateway/internal/history/query.go` - Modify: `go/vehicle-gateway/internal/history/query.go`
- Modify: `go/vehicle-gateway/internal/history/query_test.go` - Modify: `go/vehicle-gateway/internal/history/query_test.go`
- [ ] **Step 1: Write the failing test** - [x] **Step 1: Write the failing test**
Update `TestMileagePointHandlerReturnsMileageByVehicleKey` in `go/vehicle-gateway/internal/history/query_test.go` so it expects `vehicle_locations`, not `vehicle_mileage_points`: Update `TestMileagePointHandlerReturnsMileageByVehicleKey` in `go/vehicle-gateway/internal/history/query_test.go` so it expects `vehicle_locations`, not `vehicle_mileage_points`:
@@ -51,7 +51,7 @@ mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km,
}).AddRow("2026-07-02 10:00:00.000", "event-1", "frame-1", "2026-07-02 10:00:01.000", 8792.8, 8.0, 121.07764, 31.22436, "JT808", "JT808:13307811350", "VIN001", "13307811350", "dev001")) }).AddRow("2026-07-02 10:00:00.000", "event-1", "frame-1", "2026-07-02 10:00:01.000", 8792.8, 8.0, 121.07764, 31.22436, "JT808", "JT808:13307811350", "VIN001", "13307811350", "dev001"))
``` ```
- [ ] **Step 2: Run test to verify it fails** - [x] **Step 2: Run test to verify it fails**
Run: Run:
@@ -62,7 +62,7 @@ go test ./internal/history -run 'TestMileagePointHandlerReturnsMileageByVehicleK
Expected: FAIL because `MileagePointRepository.tableName()` still returns `vehicle_mileage_points`. Expected: FAIL because `MileagePointRepository.tableName()` still returns `vehicle_mileage_points`.
- [ ] **Step 3: Point mileage repository at location table** - [x] **Step 3: Point mileage repository at location table**
Change `MileagePointRepository.tableName()` in `go/vehicle-gateway/internal/history/query.go`: Change `MileagePointRepository.tableName()` in `go/vehicle-gateway/internal/history/query.go`:
@@ -108,7 +108,7 @@ func mileagePointWhere(query MileagePointQuery) []string {
} }
``` ```
- [ ] **Step 4: Run tests** - [x] **Step 4: Run tests**
Run: Run:
@@ -119,7 +119,7 @@ go test ./internal/history -count=1
Expected: PASS. Expected: PASS.
- [ ] **Step 5: Commit** - [x] **Step 5: Commit**
```bash ```bash
git add go/vehicle-gateway/internal/history/query.go go/vehicle-gateway/internal/history/query_test.go git add go/vehicle-gateway/internal/history/query.go go/vehicle-gateway/internal/history/query_test.go
@@ -133,7 +133,7 @@ git commit -m "refactor(go): read mileage points from locations"
- Modify: `go/vehicle-gateway/internal/history/writer.go` - Modify: `go/vehicle-gateway/internal/history/writer.go`
- Modify: `go/vehicle-gateway/internal/history/writer_test.go` - Modify: `go/vehicle-gateway/internal/history/writer_test.go`
- [ ] **Step 1: Write the failing tests** - [x] **Step 1: Write the failing tests**
Update `go/vehicle-gateway/internal/history/writer_test.go`: Update `go/vehicle-gateway/internal/history/writer_test.go`:
@@ -151,7 +151,7 @@ if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
} }
``` ```
- [ ] **Step 2: Run test to verify it fails** - [x] **Step 2: Run test to verify it fails**
Run: Run:
@@ -162,7 +162,7 @@ go test ./internal/history -run 'TestWriter' -count=1
Expected: FAIL because schema and writer still create/write `vehicle_mileage_points`. Expected: FAIL because schema and writer still create/write `vehicle_mileage_points`.
- [ ] **Step 3: Remove duplicate write path** - [x] **Step 3: Remove duplicate write path**
Change `AppendAll` in `go/vehicle-gateway/internal/history/writer.go`: Change `AppendAll` in `go/vehicle-gateway/internal/history/writer.go`:
@@ -177,7 +177,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro
Remove `vehicle_mileage_points` from `SchemaStatements` in `go/vehicle-gateway/internal/history/schema.go`. Keep `AppendMileagePoint` only if tests still use it directly; otherwise remove the function and its helper after compile errors guide cleanup. Remove `vehicle_mileage_points` from `SchemaStatements` in `go/vehicle-gateway/internal/history/schema.go`. Keep `AppendMileagePoint` only if tests still use it directly; otherwise remove the function and its helper after compile errors guide cleanup.
- [ ] **Step 4: Run tests** - [x] **Step 4: Run tests**
Run: Run:
@@ -188,7 +188,7 @@ go test ./internal/history ./cmd/history-writer -count=1
Expected: PASS. Expected: PASS.
- [ ] **Step 5: Commit** - [x] **Step 5: Commit**
```bash ```bash
git add go/vehicle-gateway/internal/history/schema.go go/vehicle-gateway/internal/history/writer.go go/vehicle-gateway/internal/history/writer_test.go git add go/vehicle-gateway/internal/history/schema.go go/vehicle-gateway/internal/history/writer.go go/vehicle-gateway/internal/history/writer_test.go

View File

@@ -369,9 +369,9 @@ func (r *LocationRepository) tableName() string {
func (r *MileagePointRepository) tableName() string { func (r *MileagePointRepository) tableName() string {
if r.database == "" { if r.database == "" {
return "vehicle_mileage_points" return "vehicle_locations"
} }
return r.database + ".vehicle_mileage_points" return r.database + ".vehicle_locations"
} }
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery { func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
@@ -676,7 +676,7 @@ func locationWhere(query LocationQuery) []string {
} }
func mileagePointWhere(query MileagePointQuery) []string { func mileagePointWhere(query MileagePointQuery) []string {
var where []string where := []string{"total_mileage_km IS NOT NULL"}
add := func(clause string) { add := func(clause string) {
where = append(where, clause) where = append(where, clause)
} }

View File

@@ -293,9 +293,9 @@ func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
t.Fatalf("sqlmock.New() error = %v", err) t.Fatalf("sqlmock.New() error = %v", err)
} }
defer db.Close() defer db.Close()
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points"). mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19)) WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'"). mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.vehicle_locations").
WillReturnRows(sqlmock.NewRows([]string{ WillReturnRows(sqlmock.NewRows([]string{
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude", "ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
"protocol", "vehicle_key", "vin", "phone", "device_id", "protocol", "vehicle_key", "vin", "phone", "device_id",
@@ -376,7 +376,7 @@ func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
} }
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) { func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{ sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_locations", MileagePointQuery{
Protocol: "JT808", Protocol: "JT808",
VehicleKey: "JT808:013307811350", VehicleKey: "JT808:013307811350",
DateFrom: "2026-07-02 00:00:00", DateFrom: "2026-07-02 00:00:00",
@@ -388,6 +388,8 @@ func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
t.Fatalf("expected no query args for TDengine, got %#v", args) t.Fatalf("expected no query args for TDengine, got %#v", args)
} }
for _, want := range []string{ for _, want := range []string{
"FROM lingniu_vehicle_ts.vehicle_locations",
"total_mileage_km IS NOT NULL",
"protocol = 'JT808'", "protocol = 'JT808'",
"vehicle_key = 'JT808:013307811350'", "vehicle_key = 'JT808:013307811350'",
"ts >= '2026-07-01 16:00:00'", "ts >= '2026-07-01 16:00:00'",

View File

@@ -66,22 +66,6 @@ func SchemaStatements(database string) []string {
vin NCHAR(32), vin NCHAR(32),
phone NCHAR(32), phone NCHAR(32),
device_id NCHAR(64) device_id NCHAR(64)
)`,
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
ts TIMESTAMP,
event_id NCHAR(64),
frame_id NCHAR(64),
received_at TIMESTAMP,
total_mileage_km DOUBLE,
speed_kmh DOUBLE,
longitude DOUBLE,
latitude DOUBLE
) TAGS (
protocol NCHAR(32),
vehicle_key NCHAR(64),
vin NCHAR(32),
phone NCHAR(32),
device_id NCHAR(64)
)`, )`,
} }
} }

View File

@@ -62,10 +62,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro
if err := w.AppendRawFrame(ctx, env); err != nil { if err := w.AppendRawFrame(ctx, env); err != nil {
return err return err
} }
if err := w.AppendLocation(ctx, env); err != nil { return w.AppendLocation(ctx, env)
return err
}
return w.AppendMileagePoint(ctx, env)
} }
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error { func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
@@ -121,21 +118,6 @@ VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
return err return err
} }
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
if !ok {
return nil
}
table := tableName("mil", env)
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
return err
}
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
return err
}
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error { func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
key := stable + "." + table key := stable + "." + table
w.cache.mu.Lock() w.cache.mu.Lock()
@@ -259,20 +241,6 @@ func locationValues(env envelope.FrameEnvelope, longitude float64, latitude floa
} }
} }
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
received := millis(env.ReceivedAtMS)
return []any{
eventTimeOrReceived(env),
env.StableEventID(),
frameID(env),
received,
totalMileage,
floatFieldOrNil(env, envelope.FieldSpeedKMH),
floatFieldOrNil(env, envelope.FieldLongitude),
floatFieldOrNil(env, envelope.FieldLatitude),
}
}
func frameID(env envelope.FrameEnvelope) string { func frameID(env envelope.FrameEnvelope) string {
return "go_" + env.StableEventID() return "go_" + env.StableEventID()
} }

View File

@@ -11,14 +11,17 @@ import (
func TestSchemaStatementsCreateCoreStables(t *testing.T) { func TestSchemaStatementsCreateCoreStables(t *testing.T) {
statements := strings.Join(SchemaStatements("test_ts"), "\n") statements := strings.Join(SchemaStatements("test_ts"), "\n")
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} { for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations"} {
if !strings.Contains(statements, want) { if !strings.Contains(statements, want) {
t.Fatalf("schema missing %q:\n%s", want, statements) t.Fatalf("schema missing %q:\n%s", want, statements)
} }
} }
if strings.Contains(statements, "vehicle_mileage_points") {
t.Fatalf("schema should not create duplicate mileage stable:\n%s", statements)
}
} }
func TestWriterAppendsRawLocationAndMileage(t *testing.T) { func TestWriterAppendsRawAndLocationOnly(t *testing.T) {
exec := &recordingExec{} exec := &recordingExec{}
writer := NewWriter(exec) writer := NewWriter(exec)
env := sampleEnvelope() env := sampleEnvelope()
@@ -33,8 +36,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 { if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
t.Fatalf("location child create count = %d", got) t.Fatalf("location child create count = %d", got)
} }
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 { if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 0 {
t.Fatalf("mileage child create count = %d", got) t.Fatalf("duplicate mileage child create count = %d", got)
} }
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
t.Fatalf("raw insert count = %d", got) t.Fatalf("raw insert count = %d", got)
@@ -42,8 +45,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 { if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
t.Fatalf("location insert count = %d", got) t.Fatalf("location insert count = %d", got)
} }
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 { if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
t.Fatalf("mileage insert count = %d", got) t.Fatalf("duplicate mileage insert count = %d", got)
} }
rawInsert := findSQL(exec.calls, "INSERT INTO raw_") rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") { if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {