shishengliang/lingniu-vehicle-ingest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: snapshot production code before Apple Design UI refinement

Browse Source
This commit is contained in:
lingniu
2026-09-17 18:05:57 +08:00
parent 9f0a87f49e
commit f7e7176f88
90 changed files with 8990 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go
+2 -107
View File
@@ -633,111 +633,6 @@ WHERE m.protocol = 'GB32960' AND m.daily_mileage_km > 0
}
func applyMigration(ctx context.Context, conn *sql.Conn) error {
tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
if err != nil {
return err
}
defer tx.Rollback()
if _, err := tx.ExecContext(ctx, `
INSERT INTO vehicle_data_source
(protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind,
trust_priority, enabled, first_seen_at, latest_seen_at, remark)
VALUES ('GB32960', ?, 'lingniu_prod.ln_vehicle_day_mileage', ?, ?, 'UNKNOWN', 1000, 1,
'2024-10-17 00:00:00', '2026-07-20 23:59:59',
'仅用于迁移 2026-07-21 之前缺失或为 0 的 GB32960 日里程')
ON DUPLICATE KEY UPDATE
latest_source_endpoint = VALUES(latest_source_endpoint),
platform_name = VALUES(platform_name),
source_code = VALUES(source_code),
source_kind = VALUES(source_kind),
trust_priority = VALUES(trust_priority),
enabled = 1,
remark = VALUES(remark)`, legacySourceIP, legacyPlatformName, legacySourceCode); err != nil {
return fmt.Errorf("upsert legacy data source: %w", err)
}
var sourceID int64
if err := tx.QueryRowContext(ctx, `SELECT id FROM vehicle_data_source WHERE protocol = 'GB32960' AND source_ip = ?`, legacySourceIP).Scan(&sourceID); err != nil {
return fmt.Errorf("lookup legacy data source: %w", err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO vehicle_daily_mileage_source
(vin, stat_date, protocol, source_key, source_ip, source_endpoint, platform_name,
first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count,
first_event_time, latest_event_time, quality_status, quality_reason, is_selected)
SELECT
st.vin, st.stat_date, 'GB32960', ?, ?, 'lingniu_prod.ln_vehicle_day_mileage', ?,
CASE WHEN st.total_mileage_km IS NOT NULL AND st.total_mileage_km >= st.daily_mileage_km
THEN st.total_mileage_km - st.daily_mileage_km ELSE NULL END,
st.total_mileage_km, st.daily_mileage_km, 1,
TIMESTAMP(st.stat_date, '00:00:00'), TIMESTAMP(st.stat_date, '23:59:59'),
'OK', ?, 0
FROM tmp_legacy_mileage_stage st
LEFT JOIN vehicle_daily_mileage m
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ?
ON DUPLICATE KEY UPDATE
updated_at = IF(
NOT (vehicle_daily_mileage_source.first_total_mileage_km <=> VALUES(first_total_mileage_km))
OR NOT (vehicle_daily_mileage_source.latest_total_mileage_km <=> VALUES(latest_total_mileage_km))
OR NOT (vehicle_daily_mileage_source.daily_mileage_km <=> VALUES(daily_mileage_km))
OR NOT (vehicle_daily_mileage_source.quality_status <=> VALUES(quality_status))
OR NOT (vehicle_daily_mileage_source.quality_reason <=> VALUES(quality_reason)),
CURRENT_TIMESTAMP,
vehicle_daily_mileage_source.updated_at
),
source_ip = VALUES(source_ip),
source_endpoint = VALUES(source_endpoint),
platform_name = VALUES(platform_name),
first_total_mileage_km = VALUES(first_total_mileage_km),
latest_total_mileage_km = VALUES(latest_total_mileage_km),
daily_mileage_km = VALUES(daily_mileage_km),
sample_count = VALUES(sample_count),
first_event_time = VALUES(first_event_time),
latest_event_time = VALUES(latest_event_time),
quality_status = VALUES(quality_status),
quality_reason = VALUES(quality_reason)`, legacySourceKey, legacySourceIP, legacyPlatformName, legacyQuality, sourceID); err != nil {
return fmt.Errorf("upsert legacy mileage candidates: %w", err)
}
if _, err := tx.ExecContext(ctx, `
INSERT INTO vehicle_daily_mileage
(vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km)
SELECT st.vin, st.stat_date, 'GB32960', ?, st.daily_mileage_km, st.total_mileage_km
FROM tmp_legacy_mileage_stage st
LEFT JOIN vehicle_daily_mileage m
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ?
ON DUPLICATE KEY UPDATE
updated_at = IF(
vehicle_daily_mileage.daily_mileage_km = 0
OR (
(vehicle_daily_mileage.source_id <=> VALUES(source_id))
AND (
NOT (vehicle_daily_mileage.daily_mileage_km <=> VALUES(daily_mileage_km))
OR NOT (vehicle_daily_mileage.latest_total_mileage_km <=> VALUES(latest_total_mileage_km))
)
),
CURRENT_TIMESTAMP,
vehicle_daily_mileage.updated_at
),
source_id = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(source_id), vehicle_daily_mileage.source_id),
latest_total_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(latest_total_mileage_km), vehicle_daily_mileage.latest_total_mileage_km),
daily_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(daily_mileage_km), vehicle_daily_mileage.daily_mileage_km)`, sourceID, sourceID); err != nil {
return fmt.Errorf("upsert final daily mileage: %w", err)
}
if _, err := tx.ExecContext(ctx, `
UPDATE vehicle_daily_mileage_source s
JOIN tmp_legacy_mileage_stage st
ON st.vin = s.vin AND st.stat_date = s.stat_date
JOIN vehicle_daily_mileage m
ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960'
SET s.is_selected = CASE WHEN s.source_key = ? THEN 1 ELSE 0 END
WHERE s.protocol = 'GB32960'
AND m.source_id = ?
AND ABS(m.daily_mileage_km - st.daily_mileage_km) <= 0.0005`, legacySourceKey, sourceID); err != nil {
return fmt.Errorf("mark legacy mileage selections: %w", err)
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit migration: %w", err)
}
return nil
// The legacy table has no protocol provenance. Never label its rows GB32960.
return errors.New("legacy mileage migration disabled: source rows have no verified protocol; retain the legacy table without projecting protocol mileage")
}
go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go
+9
View File
@@ -1,7 +1,9 @@
package main
import (
"context"
"database/sql"
"strings"
"testing"
)
@@ -54,3 +56,10 @@ func TestNullableTotalMileageConvertsMetersToKM(t *testing.T) {
t.Fatalf("invalid mileage should remain nil, got %#v", got)
}
}
func TestApplyRejectsUnknownProtocolBeforeDatabaseAccess(t *testing.T) {
err := applyMigration(context.Background(), nil)
if err == nil || !strings.Contains(err.Error(), "no verified protocol") {
t.Fatalf("unexpected error: %v", err)
}
}
go/vehicle-gateway/cmd/mileage-reproject/main.go
+46
View File
@@ -0,0 +1,46 @@
// Reproject explicitly listed, backed-up mileage days using the live writer SQL.
package main
import (
"context"
"database/sql"
"encoding/json"
"flag"
"fmt"
_ "github.com/go-sql-driver/mysql"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope"
"lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats"
"os"
"time"
)
func main() {
input := flag.String("input", "", "JSON array of vin/date/protocol")
flag.Parse()
b, e := os.ReadFile(*input)
must(e)
var rows []struct {
VIN string `json:"vin"`
Date string `json:"date"`
Protocol string `json:"protocol"`
}
must(json.Unmarshal(b, &rows))
db, e := sql.Open("mysql", os.Getenv("MYSQL_DSN"))
must(e)
defer db.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
defer cancel()
for i, r := range rows {
must(stats.ProjectDailyMileage(ctx, db, r.VIN, r.Date, envelope.Protocol(r.Protocol)))
if (i+1)%500 == 0 {
fmt.Printf("projected=%d/%d\n", i+1, len(rows))
}
}
fmt.Printf("completed=%d\n", len(rows))
}
func must(e error) {
if e != nil {
fmt.Fprintln(os.Stderr, e)
os.Exit(1)
}
}
go/vehicle-gateway/cmd/stats-backfill/main_test.go
+15 -15
View File
@@ -190,7 +190,7 @@ func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) {
}
}
func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testing.T) {
func TestAggregateFromDailySourceUsesHistoricalBoundaryAfterOfflineGap(t *testing.T) {
current := dailySourceLast{
VIN: "LMRKH9AC2R1004087",
SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"),
@@ -213,13 +213,13 @@ func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testin
agg := aggregateFromDailySource("2026-07-08", envelope.ProtocolYutongMQTT, current, previous, true)
if agg.FirstKM != 120778 || agg.LatestKM != 120788 {
if agg.FirstKM != 120672 || agg.LatestKM != 120788 {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS {
if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason)
}
if agg.Count != 15 {
@@ -227,7 +227,7 @@ func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testin
}
}
func TestAggregateFromDailySourceIgnoresHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) {
func TestAggregateFromDailySourceRejectsHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) {
loc := time.FixedZone("Asia/Shanghai", 8*3600)
current := dailySourceLast{
VIN: "LNXNEGRR6SR319464",
@@ -249,10 +249,10 @@ func TestAggregateFromDailySourceIgnoresHistoricalBaselineJumpAcrossOfflineGap(t
agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true)
if agg.FirstKM != current.FirstTotalKM || agg.LatestKM != current.TotalKM {
if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM {
t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(current.FirstTS) || !agg.LatestEventTime.Equal(current.TS) {
if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) {
t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime)
}
if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" {
@@ -286,7 +286,7 @@ func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testin
}
}
func TestBuildLastDiffAggregatesStartsFreshBoundaryAfterEmptyDays(t *testing.T) {
func TestBuildLastDiffAggregatesRetainsBoundaryAfterEmptyDays(t *testing.T) {
tdDB, mock, err := sqlmock.New()
if err != nil {
t.Fatalf("sqlmock.New() error = %v", err)
@@ -334,11 +334,11 @@ func TestBuildLastDiffAggregatesStartsFreshBoundaryAfterEmptyDays(t *testing.T)
if agg == nil {
t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates))
}
if agg.FirstKM != 120 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 120 -> 120", agg.FirstKM, agg.LatestKM)
if agg.FirstKM != 100 || agg.LatestKM != 120 {
t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM)
}
if !agg.FirstEventTime.Equal(dayThreeTS) || agg.QualityReason != stats.QualityReasonCurrentDayFirst {
t.Fatalf("baseline = %v reason=%q, want day-three current-day baseline", agg.FirstEventTime, agg.QualityReason)
if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical {
t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason)
}
if err := mock.ExpectationsWereMet(); err != nil {
t.Fatalf("sql expectations: %v", err)
@@ -602,10 +602,10 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes
if agg == nil {
t.Fatal("missing realtime-location fallback aggregate")
}
if agg.FirstKM != 120 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayThreeTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 120@day-three -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) {
t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM)
}
if agg.QualityReason != "realtime_location_fallback_current_day_first_baseline" {
if agg.QualityReason != "realtime_location_fallback_historical_baseline" {
t.Fatalf("quality reason = %q", agg.QualityReason)
}
if err := mysqlMock.ExpectationsWereMet(); err != nil {