From 0bbe87e6fc459fe0c8275a48170ec9f13dfd0401 Mon Sep 17 00:00:00 2001 From: lingniu Date: Thu, 2 Jul 2026 19:42:53 +0800 Subject: [PATCH] refactor(go): stop duplicate mileage history writes --- docs/architecture/storage-minimal-contract.md | 4 +-- .../2026-07-02-storage-simplification.md | 20 +++++------ go/vehicle-gateway/internal/history/query.go | 6 ++-- .../internal/history/query_test.go | 8 +++-- go/vehicle-gateway/internal/history/schema.go | 16 --------- go/vehicle-gateway/internal/history/writer.go | 34 +------------------ .../internal/history/writer_test.go | 15 ++++---- 7 files changed, 30 insertions(+), 73 deletions(-) diff --git a/docs/architecture/storage-minimal-contract.md b/docs/architecture/storage-minimal-contract.md index 2cb6fc30..923fa575 100644 --- a/docs/architecture/storage-minimal-contract.md +++ b/docs/architecture/storage-minimal-contract.md @@ -31,8 +31,8 @@ ## 当前应收敛的重复点 1. `vehicle_mileage_points` 与 `vehicle_locations.total_mileage_km` 重复。 - - 目标：停止写入 `vehicle_mileage_points`。 - - 兼容：`/api/history/mileage-points` 改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL`。 + - 状态：Go 写入链路已停止创建和写入 `vehicle_mileage_points`。 + - 兼容：`/api/history/mileage-points` 已改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL`。 - 删除：确认线上无直接读表后，再删除 TDengine stable 和子表。 2. `raw_frames.fields_json` 与 `raw_frames.parsed_json`、`vehicle_locations` 重复。 diff --git a/docs/superpowers/plans/2026-07-02-storage-simplification.md b/docs/superpowers/plans/2026-07-02-storage-simplification.md index 21e01206..cc5fb65d 100644 --- a/docs/superpowers/plans/2026-07-02-storage-simplification.md +++ b/docs/superpowers/plans/2026-07-02-storage-simplification.md @@ -37,7 +37,7 @@ - Modify: `go/vehicle-gateway/internal/history/query.go` - Modify: `go/vehicle-gateway/internal/history/query_test.go` -- [ ] **Step 1: Write the failing test** +- [x] **Step 1: Write the failing test** Update `TestMileagePointHandlerReturnsMileageByVehicleKey` in `go/vehicle-gateway/internal/history/query_test.go` so it expects `vehicle_locations`, not `vehicle_mileage_points`: @@ -51,7 +51,7 @@ mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km, }).AddRow("2026-07-02 10:00:00.000", "event-1", "frame-1", "2026-07-02 10:00:01.000", 8792.8, 8.0, 121.07764, 31.22436, "JT808", "JT808:13307811350", "VIN001", "13307811350", "dev001")) ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: @@ -62,7 +62,7 @@ go test ./internal/history -run 'TestMileagePointHandlerReturnsMileageByVehicleK Expected: FAIL because `MileagePointRepository.tableName()` still returns `vehicle_mileage_points`. -- [ ] **Step 3: Point mileage repository at location table** +- [x] **Step 3: Point mileage repository at location table** Change `MileagePointRepository.tableName()` in `go/vehicle-gateway/internal/history/query.go`: @@ -108,7 +108,7 @@ func mileagePointWhere(query MileagePointQuery) []string { } ``` -- [ ] **Step 4: Run tests** +- [x] **Step 4: Run tests** Run: @@ -119,7 +119,7 @@ go test ./internal/history -count=1 Expected: PASS. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add go/vehicle-gateway/internal/history/query.go go/vehicle-gateway/internal/history/query_test.go @@ -133,7 +133,7 @@ git commit -m "refactor(go): read mileage points from locations" - Modify: `go/vehicle-gateway/internal/history/writer.go` - Modify: `go/vehicle-gateway/internal/history/writer_test.go` -- [ ] **Step 1: Write the failing tests** +- [x] **Step 1: Write the failing tests** Update `go/vehicle-gateway/internal/history/writer_test.go`: @@ -151,7 +151,7 @@ if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 { } ``` -- [ ] **Step 2: Run test to verify it fails** +- [x] **Step 2: Run test to verify it fails** Run: @@ -162,7 +162,7 @@ go test ./internal/history -run 'TestWriter' -count=1 Expected: FAIL because schema and writer still create/write `vehicle_mileage_points`. -- [ ] **Step 3: Remove duplicate write path** +- [x] **Step 3: Remove duplicate write path** Change `AppendAll` in `go/vehicle-gateway/internal/history/writer.go`: @@ -177,7 +177,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro Remove `vehicle_mileage_points` from `SchemaStatements` in `go/vehicle-gateway/internal/history/schema.go`. Keep `AppendMileagePoint` only if tests still use it directly; otherwise remove the function and its helper after compile errors guide cleanup. -- [ ] **Step 4: Run tests** +- [x] **Step 4: Run tests** Run: @@ -188,7 +188,7 @@ go test ./internal/history ./cmd/history-writer -count=1 Expected: PASS. -- [ ] **Step 5: Commit** +- [x] **Step 5: Commit** ```bash git add go/vehicle-gateway/internal/history/schema.go go/vehicle-gateway/internal/history/writer.go go/vehicle-gateway/internal/history/writer_test.go diff --git a/go/vehicle-gateway/internal/history/query.go b/go/vehicle-gateway/internal/history/query.go index 7dfb0358..40a46d7b 100644 --- a/go/vehicle-gateway/internal/history/query.go +++ b/go/vehicle-gateway/internal/history/query.go @@ -369,9 +369,9 @@ func (r *LocationRepository) tableName() string { func (r *MileagePointRepository) tableName() string { if r.database == "" { - return "vehicle_mileage_points" + return "vehicle_locations" } - return r.database + ".vehicle_mileage_points" + return r.database + ".vehicle_locations" } func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery { @@ -676,7 +676,7 @@ func locationWhere(query LocationQuery) []string { } func mileagePointWhere(query MileagePointQuery) []string { - var where []string + where := []string{"total_mileage_km IS NOT NULL"} add := func(clause string) { where = append(where, clause) } diff --git a/go/vehicle-gateway/internal/history/query_test.go b/go/vehicle-gateway/internal/history/query_test.go index 641826ee..fc60607f 100644 --- a/go/vehicle-gateway/internal/history/query_test.go +++ b/go/vehicle-gateway/internal/history/query_test.go @@ -293,9 +293,9 @@ func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) { t.Fatalf("sqlmock.New() error = %v", err) } defer db.Close() - mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points"). + mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations"). WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19)) - mock.ExpectQuery("vehicle_key = 'JT808:013307811350'"). + mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.vehicle_locations"). WillReturnRows(sqlmock.NewRows([]string{ "ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude", "protocol", "vehicle_key", "vin", "phone", "device_id", @@ -376,7 +376,7 @@ func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) { } func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) { - sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{ + sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_locations", MileagePointQuery{ Protocol: "JT808", VehicleKey: "JT808:013307811350", DateFrom: "2026-07-02 00:00:00", @@ -388,6 +388,8 @@ func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) { t.Fatalf("expected no query args for TDengine, got %#v", args) } for _, want := range []string{ + "FROM lingniu_vehicle_ts.vehicle_locations", + "total_mileage_km IS NOT NULL", "protocol = 'JT808'", "vehicle_key = 'JT808:013307811350'", "ts >= '2026-07-01 16:00:00'", diff --git a/go/vehicle-gateway/internal/history/schema.go b/go/vehicle-gateway/internal/history/schema.go index e84cf54b..3f090207 100644 --- a/go/vehicle-gateway/internal/history/schema.go +++ b/go/vehicle-gateway/internal/history/schema.go @@ -66,22 +66,6 @@ func SchemaStatements(database string) []string { vin NCHAR(32), phone NCHAR(32), device_id NCHAR(64) -)`, - `CREATE STABLE IF NOT EXISTS vehicle_mileage_points ( - ts TIMESTAMP, - event_id NCHAR(64), - frame_id NCHAR(64), - received_at TIMESTAMP, - total_mileage_km DOUBLE, - speed_kmh DOUBLE, - longitude DOUBLE, - latitude DOUBLE -) TAGS ( - protocol NCHAR(32), - vehicle_key NCHAR(64), - vin NCHAR(32), - phone NCHAR(32), - device_id NCHAR(64) )`, } } diff --git a/go/vehicle-gateway/internal/history/writer.go b/go/vehicle-gateway/internal/history/writer.go index 99ba983b..299f9fed 100644 --- a/go/vehicle-gateway/internal/history/writer.go +++ b/go/vehicle-gateway/internal/history/writer.go @@ -62,10 +62,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro if err := w.AppendRawFrame(ctx, env); err != nil { return err } - if err := w.AppendLocation(ctx, env); err != nil { - return err - } - return w.AppendMileagePoint(ctx, env) + return w.AppendLocation(ctx, env) } func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error { @@ -121,21 +118,6 @@ VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude)))) return err } -func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error { - totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM) - if !ok { - return nil - } - table := tableName("mil", env) - if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil { - return err - } - _, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s - (ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude) -VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage)))) - return err -} - func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error { key := stable + "." + table w.cache.mu.Lock() @@ -259,20 +241,6 @@ func locationValues(env envelope.FrameEnvelope, longitude float64, latitude floa } } -func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any { - received := millis(env.ReceivedAtMS) - return []any{ - eventTimeOrReceived(env), - env.StableEventID(), - frameID(env), - received, - totalMileage, - floatFieldOrNil(env, envelope.FieldSpeedKMH), - floatFieldOrNil(env, envelope.FieldLongitude), - floatFieldOrNil(env, envelope.FieldLatitude), - } -} - func frameID(env envelope.FrameEnvelope) string { return "go_" + env.StableEventID() } diff --git a/go/vehicle-gateway/internal/history/writer_test.go b/go/vehicle-gateway/internal/history/writer_test.go index 18d18372..d5bf5b72 100644 --- a/go/vehicle-gateway/internal/history/writer_test.go +++ b/go/vehicle-gateway/internal/history/writer_test.go @@ -11,14 +11,17 @@ import ( func TestSchemaStatementsCreateCoreStables(t *testing.T) { statements := strings.Join(SchemaStatements("test_ts"), "\n") - for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} { + for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations"} { if !strings.Contains(statements, want) { t.Fatalf("schema missing %q:\n%s", want, statements) } } + if strings.Contains(statements, "vehicle_mileage_points") { + t.Fatalf("schema should not create duplicate mileage stable:\n%s", statements) + } } -func TestWriterAppendsRawLocationAndMileage(t *testing.T) { +func TestWriterAppendsRawAndLocationOnly(t *testing.T) { exec := &recordingExec{} writer := NewWriter(exec) env := sampleEnvelope() @@ -33,8 +36,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) { if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 { t.Fatalf("location child create count = %d", got) } - if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 { - t.Fatalf("mileage child create count = %d", got) + if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 0 { + t.Fatalf("duplicate mileage child create count = %d", got) } if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 { t.Fatalf("raw insert count = %d", got) @@ -42,8 +45,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) { if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 { t.Fatalf("location insert count = %d", got) } - if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 { - t.Fatalf("mileage insert count = %d", got) + if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 { + t.Fatalf("duplicate mileage insert count = %d", got) } rawInsert := findSQL(exec.calls, "INSERT INTO raw_") if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {