refactor(go): stop duplicate mileage history writes
This commit is contained in:
@@ -31,8 +31,8 @@
|
|||||||
## 当前应收敛的重复点
|
## 当前应收敛的重复点
|
||||||
|
|
||||||
1. `vehicle_mileage_points` 与 `vehicle_locations.total_mileage_km` 重复。
|
1. `vehicle_mileage_points` 与 `vehicle_locations.total_mileage_km` 重复。
|
||||||
- 目标:停止写入 `vehicle_mileage_points`。
|
- 状态:Go 写入链路已停止创建和写入 `vehicle_mileage_points`。
|
||||||
- 兼容:`/api/history/mileage-points` 改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL`。
|
- 兼容:`/api/history/mileage-points` 已改为从 `vehicle_locations` 读取 `total_mileage_km IS NOT NULL`。
|
||||||
- 删除:确认线上无直接读表后,再删除 TDengine stable 和子表。
|
- 删除:确认线上无直接读表后,再删除 TDengine stable 和子表。
|
||||||
|
|
||||||
2. `raw_frames.fields_json` 与 `raw_frames.parsed_json`、`vehicle_locations` 重复。
|
2. `raw_frames.fields_json` 与 `raw_frames.parsed_json`、`vehicle_locations` 重复。
|
||||||
|
|||||||
@@ -37,7 +37,7 @@
|
|||||||
- Modify: `go/vehicle-gateway/internal/history/query.go`
|
- Modify: `go/vehicle-gateway/internal/history/query.go`
|
||||||
- Modify: `go/vehicle-gateway/internal/history/query_test.go`
|
- Modify: `go/vehicle-gateway/internal/history/query_test.go`
|
||||||
|
|
||||||
- [ ] **Step 1: Write the failing test**
|
- [x] **Step 1: Write the failing test**
|
||||||
|
|
||||||
Update `TestMileagePointHandlerReturnsMileageByVehicleKey` in `go/vehicle-gateway/internal/history/query_test.go` so it expects `vehicle_locations`, not `vehicle_mileage_points`:
|
Update `TestMileagePointHandlerReturnsMileageByVehicleKey` in `go/vehicle-gateway/internal/history/query_test.go` so it expects `vehicle_locations`, not `vehicle_mileage_points`:
|
||||||
|
|
||||||
@@ -51,7 +51,7 @@ mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km,
|
|||||||
}).AddRow("2026-07-02 10:00:00.000", "event-1", "frame-1", "2026-07-02 10:00:01.000", 8792.8, 8.0, 121.07764, 31.22436, "JT808", "JT808:13307811350", "VIN001", "13307811350", "dev001"))
|
}).AddRow("2026-07-02 10:00:00.000", "event-1", "frame-1", "2026-07-02 10:00:01.000", 8792.8, 8.0, 121.07764, 31.22436, "JT808", "JT808:13307811350", "VIN001", "13307811350", "dev001"))
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 2: Run test to verify it fails**
|
- [x] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
@@ -62,7 +62,7 @@ go test ./internal/history -run 'TestMileagePointHandlerReturnsMileageByVehicleK
|
|||||||
|
|
||||||
Expected: FAIL because `MileagePointRepository.tableName()` still returns `vehicle_mileage_points`.
|
Expected: FAIL because `MileagePointRepository.tableName()` still returns `vehicle_mileage_points`.
|
||||||
|
|
||||||
- [ ] **Step 3: Point mileage repository at location table**
|
- [x] **Step 3: Point mileage repository at location table**
|
||||||
|
|
||||||
Change `MileagePointRepository.tableName()` in `go/vehicle-gateway/internal/history/query.go`:
|
Change `MileagePointRepository.tableName()` in `go/vehicle-gateway/internal/history/query.go`:
|
||||||
|
|
||||||
@@ -108,7 +108,7 @@ func mileagePointWhere(query MileagePointQuery) []string {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 4: Run tests**
|
- [x] **Step 4: Run tests**
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
@@ -119,7 +119,7 @@ go test ./internal/history -count=1
|
|||||||
|
|
||||||
Expected: PASS.
|
Expected: PASS.
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
- [x] **Step 5: Commit**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git add go/vehicle-gateway/internal/history/query.go go/vehicle-gateway/internal/history/query_test.go
|
git add go/vehicle-gateway/internal/history/query.go go/vehicle-gateway/internal/history/query_test.go
|
||||||
@@ -133,7 +133,7 @@ git commit -m "refactor(go): read mileage points from locations"
|
|||||||
- Modify: `go/vehicle-gateway/internal/history/writer.go`
|
- Modify: `go/vehicle-gateway/internal/history/writer.go`
|
||||||
- Modify: `go/vehicle-gateway/internal/history/writer_test.go`
|
- Modify: `go/vehicle-gateway/internal/history/writer_test.go`
|
||||||
|
|
||||||
- [ ] **Step 1: Write the failing tests**
|
- [x] **Step 1: Write the failing tests**
|
||||||
|
|
||||||
Update `go/vehicle-gateway/internal/history/writer_test.go`:
|
Update `go/vehicle-gateway/internal/history/writer_test.go`:
|
||||||
|
|
||||||
@@ -151,7 +151,7 @@ if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
- [ ] **Step 2: Run test to verify it fails**
|
- [x] **Step 2: Run test to verify it fails**
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
@@ -162,7 +162,7 @@ go test ./internal/history -run 'TestWriter' -count=1
|
|||||||
|
|
||||||
Expected: FAIL because schema and writer still create/write `vehicle_mileage_points`.
|
Expected: FAIL because schema and writer still create/write `vehicle_mileage_points`.
|
||||||
|
|
||||||
- [ ] **Step 3: Remove duplicate write path**
|
- [x] **Step 3: Remove duplicate write path**
|
||||||
|
|
||||||
Change `AppendAll` in `go/vehicle-gateway/internal/history/writer.go`:
|
Change `AppendAll` in `go/vehicle-gateway/internal/history/writer.go`:
|
||||||
|
|
||||||
@@ -177,7 +177,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro
|
|||||||
|
|
||||||
Remove `vehicle_mileage_points` from `SchemaStatements` in `go/vehicle-gateway/internal/history/schema.go`. Keep `AppendMileagePoint` only if tests still use it directly; otherwise remove the function and its helper after compile errors guide cleanup.
|
Remove `vehicle_mileage_points` from `SchemaStatements` in `go/vehicle-gateway/internal/history/schema.go`. Keep `AppendMileagePoint` only if tests still use it directly; otherwise remove the function and its helper after compile errors guide cleanup.
|
||||||
|
|
||||||
- [ ] **Step 4: Run tests**
|
- [x] **Step 4: Run tests**
|
||||||
|
|
||||||
Run:
|
Run:
|
||||||
|
|
||||||
@@ -188,7 +188,7 @@ go test ./internal/history ./cmd/history-writer -count=1
|
|||||||
|
|
||||||
Expected: PASS.
|
Expected: PASS.
|
||||||
|
|
||||||
- [ ] **Step 5: Commit**
|
- [x] **Step 5: Commit**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
git add go/vehicle-gateway/internal/history/schema.go go/vehicle-gateway/internal/history/writer.go go/vehicle-gateway/internal/history/writer_test.go
|
git add go/vehicle-gateway/internal/history/schema.go go/vehicle-gateway/internal/history/writer.go go/vehicle-gateway/internal/history/writer_test.go
|
||||||
|
|||||||
@@ -369,9 +369,9 @@ func (r *LocationRepository) tableName() string {
|
|||||||
|
|
||||||
func (r *MileagePointRepository) tableName() string {
|
func (r *MileagePointRepository) tableName() string {
|
||||||
if r.database == "" {
|
if r.database == "" {
|
||||||
return "vehicle_mileage_points"
|
return "vehicle_locations"
|
||||||
}
|
}
|
||||||
return r.database + ".vehicle_mileage_points"
|
return r.database + ".vehicle_locations"
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
|
func normalizeRawFrameQuery(query RawFrameQuery) RawFrameQuery {
|
||||||
@@ -676,7 +676,7 @@ func locationWhere(query LocationQuery) []string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func mileagePointWhere(query MileagePointQuery) []string {
|
func mileagePointWhere(query MileagePointQuery) []string {
|
||||||
var where []string
|
where := []string{"total_mileage_km IS NOT NULL"}
|
||||||
add := func(clause string) {
|
add := func(clause string) {
|
||||||
where = append(where, clause)
|
where = append(where, clause)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -293,9 +293,9 @@ func TestMileagePointHandlerReturnsMileageByVehicleKey(t *testing.T) {
|
|||||||
t.Fatalf("sqlmock.New() error = %v", err)
|
t.Fatalf("sqlmock.New() error = %v", err)
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_mileage_points").
|
mock.ExpectQuery("SELECT COUNT\\(\\*\\) FROM lingniu_vehicle_ts.vehicle_locations").
|
||||||
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
|
WillReturnRows(sqlmock.NewRows([]string{"total"}).AddRow(19))
|
||||||
mock.ExpectQuery("vehicle_key = 'JT808:013307811350'").
|
mock.ExpectQuery("SELECT ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude, protocol, vehicle_key, vin, phone, device_id FROM lingniu_vehicle_ts.vehicle_locations").
|
||||||
WillReturnRows(sqlmock.NewRows([]string{
|
WillReturnRows(sqlmock.NewRows([]string{
|
||||||
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
|
"ts", "event_id", "frame_id", "received_at", "total_mileage_km", "speed_kmh", "longitude", "latitude",
|
||||||
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
"protocol", "vehicle_key", "vin", "phone", "device_id",
|
||||||
@@ -376,7 +376,7 @@ func TestParseMessageIDSupportsDecimalAndHex(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
||||||
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_mileage_points", MileagePointQuery{
|
sqlText, args := buildMileagePointSQL("lingniu_vehicle_ts.vehicle_locations", MileagePointQuery{
|
||||||
Protocol: "JT808",
|
Protocol: "JT808",
|
||||||
VehicleKey: "JT808:013307811350",
|
VehicleKey: "JT808:013307811350",
|
||||||
DateFrom: "2026-07-02 00:00:00",
|
DateFrom: "2026-07-02 00:00:00",
|
||||||
@@ -388,6 +388,8 @@ func TestBuildMileagePointSQLUsesLiteralsForTDengine(t *testing.T) {
|
|||||||
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
t.Fatalf("expected no query args for TDengine, got %#v", args)
|
||||||
}
|
}
|
||||||
for _, want := range []string{
|
for _, want := range []string{
|
||||||
|
"FROM lingniu_vehicle_ts.vehicle_locations",
|
||||||
|
"total_mileage_km IS NOT NULL",
|
||||||
"protocol = 'JT808'",
|
"protocol = 'JT808'",
|
||||||
"vehicle_key = 'JT808:013307811350'",
|
"vehicle_key = 'JT808:013307811350'",
|
||||||
"ts >= '2026-07-01 16:00:00'",
|
"ts >= '2026-07-01 16:00:00'",
|
||||||
|
|||||||
@@ -66,22 +66,6 @@ func SchemaStatements(database string) []string {
|
|||||||
vin NCHAR(32),
|
vin NCHAR(32),
|
||||||
phone NCHAR(32),
|
phone NCHAR(32),
|
||||||
device_id NCHAR(64)
|
device_id NCHAR(64)
|
||||||
)`,
|
|
||||||
`CREATE STABLE IF NOT EXISTS vehicle_mileage_points (
|
|
||||||
ts TIMESTAMP,
|
|
||||||
event_id NCHAR(64),
|
|
||||||
frame_id NCHAR(64),
|
|
||||||
received_at TIMESTAMP,
|
|
||||||
total_mileage_km DOUBLE,
|
|
||||||
speed_kmh DOUBLE,
|
|
||||||
longitude DOUBLE,
|
|
||||||
latitude DOUBLE
|
|
||||||
) TAGS (
|
|
||||||
protocol NCHAR(32),
|
|
||||||
vehicle_key NCHAR(64),
|
|
||||||
vin NCHAR(32),
|
|
||||||
phone NCHAR(32),
|
|
||||||
device_id NCHAR(64)
|
|
||||||
)`,
|
)`,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,10 +62,7 @@ func (w *Writer) AppendAll(ctx context.Context, env envelope.FrameEnvelope) erro
|
|||||||
if err := w.AppendRawFrame(ctx, env); err != nil {
|
if err := w.AppendRawFrame(ctx, env); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := w.AppendLocation(ctx, env); err != nil {
|
return w.AppendLocation(ctx, env)
|
||||||
return err
|
|
||||||
}
|
|
||||||
return w.AppendMileagePoint(ctx, env)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
func (w *Writer) AppendRawFrame(ctx context.Context, env envelope.FrameEnvelope) error {
|
||||||
@@ -121,21 +118,6 @@ VALUES (%s)`, table, joinLiterals(locationValues(env, longitude, latitude))))
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Writer) AppendMileagePoint(ctx context.Context, env envelope.FrameEnvelope) error {
|
|
||||||
totalMileage, ok := floatField(env, envelope.FieldTotalMileageKM)
|
|
||||||
if !ok {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
table := tableName("mil", env)
|
|
||||||
if err := w.ensureChild(ctx, table, "vehicle_mileage_points", env); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
_, err := w.exec.ExecContext(ctx, fmt.Sprintf(`INSERT INTO %s
|
|
||||||
(ts, event_id, frame_id, received_at, total_mileage_km, speed_kmh, longitude, latitude)
|
|
||||||
VALUES (%s)`, table, joinLiterals(mileageValues(env, totalMileage))))
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
func (w *Writer) ensureChild(ctx context.Context, table string, stable string, env envelope.FrameEnvelope) error {
|
||||||
key := stable + "." + table
|
key := stable + "." + table
|
||||||
w.cache.mu.Lock()
|
w.cache.mu.Lock()
|
||||||
@@ -259,20 +241,6 @@ func locationValues(env envelope.FrameEnvelope, longitude float64, latitude floa
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mileageValues(env envelope.FrameEnvelope, totalMileage float64) []any {
|
|
||||||
received := millis(env.ReceivedAtMS)
|
|
||||||
return []any{
|
|
||||||
eventTimeOrReceived(env),
|
|
||||||
env.StableEventID(),
|
|
||||||
frameID(env),
|
|
||||||
received,
|
|
||||||
totalMileage,
|
|
||||||
floatFieldOrNil(env, envelope.FieldSpeedKMH),
|
|
||||||
floatFieldOrNil(env, envelope.FieldLongitude),
|
|
||||||
floatFieldOrNil(env, envelope.FieldLatitude),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func frameID(env envelope.FrameEnvelope) string {
|
func frameID(env envelope.FrameEnvelope) string {
|
||||||
return "go_" + env.StableEventID()
|
return "go_" + env.StableEventID()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,14 +11,17 @@ import (
|
|||||||
|
|
||||||
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
func TestSchemaStatementsCreateCoreStables(t *testing.T) {
|
||||||
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
statements := strings.Join(SchemaStatements("test_ts"), "\n")
|
||||||
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations", "vehicle_mileage_points"} {
|
for _, want := range []string{"CREATE DATABASE IF NOT EXISTS test_ts", "raw_frames", "vehicle_locations"} {
|
||||||
if !strings.Contains(statements, want) {
|
if !strings.Contains(statements, want) {
|
||||||
t.Fatalf("schema missing %q:\n%s", want, statements)
|
t.Fatalf("schema missing %q:\n%s", want, statements)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if strings.Contains(statements, "vehicle_mileage_points") {
|
||||||
|
t.Fatalf("schema should not create duplicate mileage stable:\n%s", statements)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
func TestWriterAppendsRawAndLocationOnly(t *testing.T) {
|
||||||
exec := &recordingExec{}
|
exec := &recordingExec{}
|
||||||
writer := NewWriter(exec)
|
writer := NewWriter(exec)
|
||||||
env := sampleEnvelope()
|
env := sampleEnvelope()
|
||||||
@@ -33,8 +36,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
|||||||
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
if got := countSQL(exec.calls, "USING vehicle_locations"); got != 1 {
|
||||||
t.Fatalf("location child create count = %d", got)
|
t.Fatalf("location child create count = %d", got)
|
||||||
}
|
}
|
||||||
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 1 {
|
if got := countSQL(exec.calls, "USING vehicle_mileage_points"); got != 0 {
|
||||||
t.Fatalf("mileage child create count = %d", got)
|
t.Fatalf("duplicate mileage child create count = %d", got)
|
||||||
}
|
}
|
||||||
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
if got := countSQL(exec.calls, "INSERT INTO raw_"); got != 1 {
|
||||||
t.Fatalf("raw insert count = %d", got)
|
t.Fatalf("raw insert count = %d", got)
|
||||||
@@ -42,8 +45,8 @@ func TestWriterAppendsRawLocationAndMileage(t *testing.T) {
|
|||||||
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
if got := countSQL(exec.calls, "INSERT INTO loc_"); got != 1 {
|
||||||
t.Fatalf("location insert count = %d", got)
|
t.Fatalf("location insert count = %d", got)
|
||||||
}
|
}
|
||||||
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 1 {
|
if got := countSQL(exec.calls, "INSERT INTO mil_"); got != 0 {
|
||||||
t.Fatalf("mileage insert count = %d", got)
|
t.Fatalf("duplicate mileage insert count = %d", got)
|
||||||
}
|
}
|
||||||
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
rawInsert := findSQL(exec.calls, "INSERT INTO raw_")
|
||||||
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
if !strings.Contains(rawInsert, "'go_") || !strings.Contains(rawInsert, "'2e") && !strings.Contains(rawInsert, "'") {
|
||||||
|
|||||||
Reference in New Issue
Block a user