From f7e7176f88b4fd49338dda8e72d701a24617fb2f Mon Sep 17 00:00:00 2001 From: lingniu Date: Thu, 17 Sep 2026 18:05:57 +0800 Subject: [PATCH] chore: snapshot production code before Apple Design UI refinement --- .../cmd/legacy-mileage-migrate/main.go | 109 +-- .../cmd/legacy-mileage-migrate/main_test.go | 9 + .../cmd/mileage-reproject/main.go | 46 + .../cmd/stats-backfill/main_test.go | 30 +- .../internal/stats/daily_metric.go | 4 +- .../internal/stats/daily_metric_test.go | 34 +- .../internal/stats/source_mileage.go | 14 +- .../internal/stats/source_mileage_test.go | 4 +- .../mileage-reconciliation-20260916/backup.py | 49 ++ .../db_access.py | 17 + .../mileage-reconciliation-20260916/repair.py | 60 ++ .../run_remote.py | 7 + .../inspect_sample.mjs | 115 +++ tmp/g7-mileage-history-20260724/build_all.mjs | 741 +++++++++++++++++ tmp/g7-mileage-history-20260724/importer.go | 787 ++++++++++++++++++ .../schema_probe.go | 66 ++ .../verify_final.mjs | 41 + tmp/gb32960-export-20260817/build_export.mjs | 364 ++++++++ tmp/gb32960-export-20260817/fetch_data.mjs | 77 ++ tmp/gb32960-export-20260817/verify_final.mjs | 15 + .../inspect_sources.mjs | 94 +++ .../normalize_compare.mjs | 378 +++++++++ .../assess_candidates.mjs | 40 + .../build_real_audit.mjs | 95 +++ tmp/hydrogen-audit-100-real/inspect_v3.mjs | 10 + .../inspect.mjs | 46 + .../build_hydrogen_audit.mjs | 496 +++++++++++ tmp/mileage-appeal-20260810/build_appeal.mjs | 446 ++++++++++ .../collect_evidence.mjs | 158 ++++ .../inspect_source.mjs | 51 ++ .../make_contact_sheet.py | 28 + tmp/mileage-appeal-20260810/verify_final.mjs | 27 + tmp/seg-mileage-import-20260907/db.py | 17 + tmp/seg-mileage-import-20260907/import.py | 80 ++ tmp/seg-mileage-import-20260907/prepare.py | 25 + tmp/seg-mileage-import-20260907/verify.py | 15 + .../build.mjs | 638 ++++++++++++++ .../apps/api/cmd/daily-geography/main.go | 133 +++ .../api/cmd/mileage-reconcile-audit/main.go | 93 +++ .../apps/api/cmd/native-alarm-repair/main.go | 174 ++++ .../api/cmd/native-alarm-repair/main_test.go | 108 +++ .../apps/api/internal/app/server.go | 6 + .../apps/api/internal/config/config.go | 2 + .../apps/api/internal/dailygeo/coordinates.go | 35 + .../apps/api/internal/dailygeo/geocoder.go | 123 +++ .../apps/api/internal/dailygeo/worker.go | 297 +++++++ .../apps/api/internal/dailygeo/worker_test.go | 138 +++ .../internal/openplatform/assets/docs.html | 2 +- .../internal/openplatform/assets/openapi.yaml | 20 +- .../openplatform/mileage_reconciliation.go | 171 ++++ .../mileage_reconciliation_test.go | 83 ++ .../apps/api/internal/openplatform/model.go | 2 + .../apps/api/internal/openplatform/service.go | 73 +- .../api/internal/openplatform/service_test.go | 50 ++ .../apps/api/internal/platform/alert.go | 3 + .../apps/api/internal/platform/alert_store.go | 14 + .../platform/alert_stream_evaluator.go | 3 + .../platform/corrected_hydrogen_test.go | 51 ++ .../platform/hydrogen_evidence_test.go | 3 + .../apps/api/internal/platform/model.go | 129 +-- .../api/internal/platform/mysql_queries.go | 11 +- .../api/internal/platform/native_alarm.go | 142 ++++ .../platform/native_alarm_description.go | 84 ++ .../internal/platform/native_alarm_test.go | 203 +++++ .../api/internal/platform/production_store.go | 7 +- .../platform/production_store_test.go | 7 +- .../internal/platform/query_builders_test.go | 2 +- .../apps/api/internal/platform/service.go | 21 + .../api/internal/platform/service_test.go | 2 +- .../api/tmp/hydrogen-audit-100-real/main.go | 681 +++++++++++++++ .../hydrogen-audit-100-real/query_capacity.go | 38 + .../hydrogen-email-audit/query_metadata.go | 97 +++ .../apps/api/tmp/standard-model-trial/main.go | 660 +++++++++++++++ .../apps/web/src/api/client.ts | 2 +- .../apps/web/src/api/types.ts | 10 +- .../apps/web/src/v2/domain/alert.ts | 2 +- .../apps/web/src/v2/domain/event.ts | 22 + .../web/src/v2/domain/mileageExport.test.ts | 19 +- .../apps/web/src/v2/domain/mileageExport.ts | 6 +- .../apps/web/src/v2/domain/mileageWorkbook.ts | 29 +- .../apps/web/src/v2/pages/AlertsPage.test.tsx | 39 + .../apps/web/src/v2/pages/AlertsPage.tsx | 28 +- .../web/src/v2/pages/StatisticsPage.test.tsx | 24 +- .../apps/web/src/v2/pages/StatisticsPage.tsx | 36 +- .../deploy/migrations/046_daily_geography.sql | 31 + .../migrations/047_native_gb32960_alarms.sql | 10 + .../docs/daily-geography-20260910.md | 56 ++ .../rollback-baseline.md | 19 + .../hydrogen-consumption-export-20260910.md | 45 + .../docs/native-gb32960-alarms.md | 39 + 90 files changed, 8990 insertions(+), 328 deletions(-) create mode 100644 go/vehicle-gateway/cmd/mileage-reproject/main.go create mode 100644 reports/mileage-reconciliation-20260916/backup.py create mode 100644 reports/mileage-reconciliation-20260916/db_access.py create mode 100644 reports/mileage-reconciliation-20260916/repair.py create mode 100644 reports/mileage-reconciliation-20260916/run_remote.py create mode 100644 tmp/g7-mileage-eval-20260724/inspect_sample.mjs create mode 100644 tmp/g7-mileage-history-20260724/build_all.mjs create mode 100644 tmp/g7-mileage-history-20260724/importer.go create mode 100644 tmp/g7-mileage-history-20260724/schema_probe.go create mode 100644 tmp/g7-mileage-history-20260724/verify_final.mjs create mode 100644 tmp/gb32960-export-20260817/build_export.mjs create mode 100644 tmp/gb32960-export-20260817/fetch_data.mjs create mode 100644 tmp/gb32960-export-20260817/verify_final.mjs create mode 100644 tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/inspect_sources.mjs create mode 100644 tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/normalize_compare.mjs create mode 100644 tmp/hydrogen-audit-100-real/assess_candidates.mjs create mode 100644 tmp/hydrogen-audit-100-real/build_real_audit.mjs create mode 100644 tmp/hydrogen-audit-100-real/inspect_v3.mjs create mode 100644 tmp/hydrogen-formula-analysis-20260826/inspect.mjs create mode 100644 tmp/hydrogen_audit_120/build_hydrogen_audit.mjs create mode 100644 tmp/mileage-appeal-20260810/build_appeal.mjs create mode 100644 tmp/mileage-appeal-20260810/collect_evidence.mjs create mode 100644 tmp/mileage-appeal-20260810/inspect_source.mjs create mode 100644 tmp/mileage-appeal-20260810/make_contact_sheet.py create mode 100644 tmp/mileage-appeal-20260810/verify_final.mjs create mode 100644 tmp/seg-mileage-import-20260907/db.py create mode 100644 tmp/seg-mileage-import-20260907/import.py create mode 100644 tmp/seg-mileage-import-20260907/prepare.py create mode 100644 tmp/seg-mileage-import-20260907/verify.py create mode 100644 tmp/vehicle-mileage-selection-019fa1ac-db39-7933-bb0e-30dd63cc25bd/build.mjs create mode 100644 vehicle-data-platform/apps/api/cmd/daily-geography/main.go create mode 100644 vehicle-data-platform/apps/api/cmd/mileage-reconcile-audit/main.go create mode 100644 vehicle-data-platform/apps/api/cmd/native-alarm-repair/main.go create mode 100644 vehicle-data-platform/apps/api/cmd/native-alarm-repair/main_test.go create mode 100644 vehicle-data-platform/apps/api/internal/dailygeo/coordinates.go create mode 100644 vehicle-data-platform/apps/api/internal/dailygeo/geocoder.go create mode 100644 vehicle-data-platform/apps/api/internal/dailygeo/worker.go create mode 100644 vehicle-data-platform/apps/api/internal/dailygeo/worker_test.go create mode 100644 vehicle-data-platform/apps/api/internal/openplatform/mileage_reconciliation.go create mode 100644 vehicle-data-platform/apps/api/internal/openplatform/mileage_reconciliation_test.go create mode 100644 vehicle-data-platform/apps/api/internal/platform/corrected_hydrogen_test.go create mode 100644 vehicle-data-platform/apps/api/internal/platform/native_alarm.go create mode 100644 vehicle-data-platform/apps/api/internal/platform/native_alarm_description.go create mode 100644 vehicle-data-platform/apps/api/internal/platform/native_alarm_test.go create mode 100644 vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/main.go create mode 100644 vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/query_capacity.go create mode 100644 vehicle-data-platform/apps/api/tmp/hydrogen-email-audit/query_metadata.go create mode 100644 vehicle-data-platform/apps/api/tmp/standard-model-trial/main.go create mode 100644 vehicle-data-platform/deploy/migrations/046_daily_geography.sql create mode 100644 vehicle-data-platform/deploy/migrations/047_native_gb32960_alarms.sql create mode 100644 vehicle-data-platform/docs/daily-geography-20260910.md create mode 100644 vehicle-data-platform/docs/design/apple-design-20260917/rollback-baseline.md create mode 100644 vehicle-data-platform/docs/hydrogen-consumption-export-20260910.md create mode 100644 vehicle-data-platform/docs/native-gb32960-alarms.md diff --git a/go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go b/go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go index b1df01c3..62950cc9 100644 --- a/go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go +++ b/go/vehicle-gateway/cmd/legacy-mileage-migrate/main.go @@ -633,111 +633,6 @@ WHERE m.protocol = 'GB32960' AND m.daily_mileage_km > 0 } func applyMigration(ctx context.Context, conn *sql.Conn) error { - tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) - if err != nil { - return err - } - defer tx.Rollback() - if _, err := tx.ExecContext(ctx, ` -INSERT INTO vehicle_data_source - (protocol, source_ip, latest_source_endpoint, platform_name, source_code, source_kind, - trust_priority, enabled, first_seen_at, latest_seen_at, remark) -VALUES ('GB32960', ?, 'lingniu_prod.ln_vehicle_day_mileage', ?, ?, 'UNKNOWN', 1000, 1, - '2024-10-17 00:00:00', '2026-07-20 23:59:59', - '仅用于迁移 2026-07-21 之前缺失或为 0 的 GB32960 日里程') -ON DUPLICATE KEY UPDATE - latest_source_endpoint = VALUES(latest_source_endpoint), - platform_name = VALUES(platform_name), - source_code = VALUES(source_code), - source_kind = VALUES(source_kind), - trust_priority = VALUES(trust_priority), - enabled = 1, - remark = VALUES(remark)`, legacySourceIP, legacyPlatformName, legacySourceCode); err != nil { - return fmt.Errorf("upsert legacy data source: %w", err) - } - var sourceID int64 - if err := tx.QueryRowContext(ctx, `SELECT id FROM vehicle_data_source WHERE protocol = 'GB32960' AND source_ip = ?`, legacySourceIP).Scan(&sourceID); err != nil { - return fmt.Errorf("lookup legacy data source: %w", err) - } - if _, err := tx.ExecContext(ctx, ` -INSERT INTO vehicle_daily_mileage_source - (vin, stat_date, protocol, source_key, source_ip, source_endpoint, platform_name, - first_total_mileage_km, latest_total_mileage_km, daily_mileage_km, sample_count, - first_event_time, latest_event_time, quality_status, quality_reason, is_selected) -SELECT - st.vin, st.stat_date, 'GB32960', ?, ?, 'lingniu_prod.ln_vehicle_day_mileage', ?, - CASE WHEN st.total_mileage_km IS NOT NULL AND st.total_mileage_km >= st.daily_mileage_km - THEN st.total_mileage_km - st.daily_mileage_km ELSE NULL END, - st.total_mileage_km, st.daily_mileage_km, 1, - TIMESTAMP(st.stat_date, '00:00:00'), TIMESTAMP(st.stat_date, '23:59:59'), - 'OK', ?, 0 -FROM tmp_legacy_mileage_stage st -LEFT JOIN vehicle_daily_mileage m - ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960' -WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ? -ON DUPLICATE KEY UPDATE - updated_at = IF( - NOT (vehicle_daily_mileage_source.first_total_mileage_km <=> VALUES(first_total_mileage_km)) - OR NOT (vehicle_daily_mileage_source.latest_total_mileage_km <=> VALUES(latest_total_mileage_km)) - OR NOT (vehicle_daily_mileage_source.daily_mileage_km <=> VALUES(daily_mileage_km)) - OR NOT (vehicle_daily_mileage_source.quality_status <=> VALUES(quality_status)) - OR NOT (vehicle_daily_mileage_source.quality_reason <=> VALUES(quality_reason)), - CURRENT_TIMESTAMP, - vehicle_daily_mileage_source.updated_at - ), - source_ip = VALUES(source_ip), - source_endpoint = VALUES(source_endpoint), - platform_name = VALUES(platform_name), - first_total_mileage_km = VALUES(first_total_mileage_km), - latest_total_mileage_km = VALUES(latest_total_mileage_km), - daily_mileage_km = VALUES(daily_mileage_km), - sample_count = VALUES(sample_count), - first_event_time = VALUES(first_event_time), - latest_event_time = VALUES(latest_event_time), - quality_status = VALUES(quality_status), - quality_reason = VALUES(quality_reason)`, legacySourceKey, legacySourceIP, legacyPlatformName, legacyQuality, sourceID); err != nil { - return fmt.Errorf("upsert legacy mileage candidates: %w", err) - } - if _, err := tx.ExecContext(ctx, ` -INSERT INTO vehicle_daily_mileage - (vin, stat_date, protocol, source_id, daily_mileage_km, latest_total_mileage_km) -SELECT st.vin, st.stat_date, 'GB32960', ?, st.daily_mileage_km, st.total_mileage_km -FROM tmp_legacy_mileage_stage st -LEFT JOIN vehicle_daily_mileage m - ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960' -WHERE m.vin IS NULL OR m.daily_mileage_km = 0 OR m.source_id = ? -ON DUPLICATE KEY UPDATE - updated_at = IF( - vehicle_daily_mileage.daily_mileage_km = 0 - OR ( - (vehicle_daily_mileage.source_id <=> VALUES(source_id)) - AND ( - NOT (vehicle_daily_mileage.daily_mileage_km <=> VALUES(daily_mileage_km)) - OR NOT (vehicle_daily_mileage.latest_total_mileage_km <=> VALUES(latest_total_mileage_km)) - ) - ), - CURRENT_TIMESTAMP, - vehicle_daily_mileage.updated_at - ), - source_id = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(source_id), vehicle_daily_mileage.source_id), - latest_total_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(latest_total_mileage_km), vehicle_daily_mileage.latest_total_mileage_km), - daily_mileage_km = IF(vehicle_daily_mileage.daily_mileage_km = 0 OR (vehicle_daily_mileage.source_id <=> VALUES(source_id)), VALUES(daily_mileage_km), vehicle_daily_mileage.daily_mileage_km)`, sourceID, sourceID); err != nil { - return fmt.Errorf("upsert final daily mileage: %w", err) - } - if _, err := tx.ExecContext(ctx, ` -UPDATE vehicle_daily_mileage_source s -JOIN tmp_legacy_mileage_stage st - ON st.vin = s.vin AND st.stat_date = s.stat_date -JOIN vehicle_daily_mileage m - ON m.vin = st.vin AND m.stat_date = st.stat_date AND m.protocol = 'GB32960' -SET s.is_selected = CASE WHEN s.source_key = ? THEN 1 ELSE 0 END -WHERE s.protocol = 'GB32960' - AND m.source_id = ? - AND ABS(m.daily_mileage_km - st.daily_mileage_km) <= 0.0005`, legacySourceKey, sourceID); err != nil { - return fmt.Errorf("mark legacy mileage selections: %w", err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit migration: %w", err) - } - return nil + // The legacy table has no protocol provenance. Never label its rows GB32960. + return errors.New("legacy mileage migration disabled: source rows have no verified protocol; retain the legacy table without projecting protocol mileage") } diff --git a/go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go b/go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go index 0300fc26..351d412c 100644 --- a/go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go +++ b/go/vehicle-gateway/cmd/legacy-mileage-migrate/main_test.go @@ -1,7 +1,9 @@ package main import ( + "context" "database/sql" + "strings" "testing" ) @@ -54,3 +56,10 @@ func TestNullableTotalMileageConvertsMetersToKM(t *testing.T) { t.Fatalf("invalid mileage should remain nil, got %#v", got) } } + +func TestApplyRejectsUnknownProtocolBeforeDatabaseAccess(t *testing.T) { + err := applyMigration(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "no verified protocol") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/go/vehicle-gateway/cmd/mileage-reproject/main.go b/go/vehicle-gateway/cmd/mileage-reproject/main.go new file mode 100644 index 00000000..e3ed44d5 --- /dev/null +++ b/go/vehicle-gateway/cmd/mileage-reproject/main.go @@ -0,0 +1,46 @@ +// Reproject explicitly listed, backed-up mileage days using the live writer SQL. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + _ "github.com/go-sql-driver/mysql" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/envelope" + "lingniu-vehicle-ingest/go/vehicle-gateway/internal/stats" + "os" + "time" +) + +func main() { + input := flag.String("input", "", "JSON array of vin/date/protocol") + flag.Parse() + b, e := os.ReadFile(*input) + must(e) + var rows []struct { + VIN string `json:"vin"` + Date string `json:"date"` + Protocol string `json:"protocol"` + } + must(json.Unmarshal(b, &rows)) + db, e := sql.Open("mysql", os.Getenv("MYSQL_DSN")) + must(e) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + for i, r := range rows { + must(stats.ProjectDailyMileage(ctx, db, r.VIN, r.Date, envelope.Protocol(r.Protocol))) + if (i+1)%500 == 0 { + fmt.Printf("projected=%d/%d\n", i+1, len(rows)) + } + } + fmt.Printf("completed=%d\n", len(rows)) +} +func must(e error) { + if e != nil { + fmt.Fprintln(os.Stderr, e) + os.Exit(1) + } +} diff --git a/go/vehicle-gateway/cmd/stats-backfill/main_test.go b/go/vehicle-gateway/cmd/stats-backfill/main_test.go index fdad4a7e..be70d55e 100644 --- a/go/vehicle-gateway/cmd/stats-backfill/main_test.go +++ b/go/vehicle-gateway/cmd/stats-backfill/main_test.go @@ -190,7 +190,7 @@ func TestDailySourceLastBuildsCandidateKeysBySourceIP(t *testing.T) { } } -func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testing.T) { +func TestAggregateFromDailySourceUsesHistoricalBoundaryAfterOfflineGap(t *testing.T) { current := dailySourceLast{ VIN: "LMRKH9AC2R1004087", SourceKey: normalizedSourceKey("YUTONG_MQTT", "", "LMRKH9AC2R1004087", "mqtt://yutong/ytforward/shln/3"), @@ -213,13 +213,13 @@ func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testin agg := aggregateFromDailySource("2026-07-08", envelope.ProtocolYutongMQTT, current, previous, true) - if agg.FirstKM != 120778 || agg.LatestKM != 120788 { + if agg.FirstKM != 120672 || agg.LatestKM != 120788 { t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM) } - if agg.FirstEventTime != current.FirstTS || agg.LatestEventTime != current.TS { + if agg.FirstEventTime != previous.TS || agg.LatestEventTime != current.TS { t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime) } - if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonCurrentDayFirst { + if agg.QualityStatus != stats.QualityOK || agg.QualityReason != stats.QualityReasonHistorical { t.Fatalf("quality = %s/%s", agg.QualityStatus, agg.QualityReason) } if agg.Count != 15 { @@ -227,7 +227,7 @@ func TestAggregateFromDailySourceUsesCurrentDayBoundaryAfterOfflineGap(t *testin } } -func TestAggregateFromDailySourceIgnoresHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) { +func TestAggregateFromDailySourceRejectsHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) current := dailySourceLast{ VIN: "LNXNEGRR6SR319464", @@ -249,10 +249,10 @@ func TestAggregateFromDailySourceIgnoresHistoricalBaselineJumpAcrossOfflineGap(t agg := aggregateFromDailySource("2026-07-12", envelope.ProtocolGB32960, current, previous, true) - if agg.FirstKM != current.FirstTotalKM || agg.LatestKM != current.TotalKM { + if agg.FirstKM != previous.TotalKM || agg.LatestKM != current.TotalKM { t.Fatalf("km range = %v -> %v", agg.FirstKM, agg.LatestKM) } - if !agg.FirstEventTime.Equal(current.FirstTS) || !agg.LatestEventTime.Equal(current.TS) { + if !agg.FirstEventTime.Equal(previous.TS) || !agg.LatestEventTime.Equal(current.TS) { t.Fatalf("event range = %v -> %v", agg.FirstEventTime, agg.LatestEventTime) } if agg.QualityStatus != stats.QualityInvalidDelta || agg.QualityReason != "outside_daily_range" { @@ -286,7 +286,7 @@ func TestAggregateFromDailySourceUsesCurrentDayFirstWhenHistoryIsEmpty(t *testin } } -func TestBuildLastDiffAggregatesStartsFreshBoundaryAfterEmptyDays(t *testing.T) { +func TestBuildLastDiffAggregatesRetainsBoundaryAfterEmptyDays(t *testing.T) { tdDB, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) @@ -334,11 +334,11 @@ func TestBuildLastDiffAggregatesStartsFreshBoundaryAfterEmptyDays(t *testing.T) if agg == nil { t.Fatalf("missing day-three aggregate; keys=%v", aggregateKeys(aggregates)) } - if agg.FirstKM != 120 || agg.LatestKM != 120 { - t.Fatalf("km range = %v -> %v, want 120 -> 120", agg.FirstKM, agg.LatestKM) + if agg.FirstKM != 100 || agg.LatestKM != 120 { + t.Fatalf("km range = %v -> %v, want 100 -> 120", agg.FirstKM, agg.LatestKM) } - if !agg.FirstEventTime.Equal(dayThreeTS) || agg.QualityReason != stats.QualityReasonCurrentDayFirst { - t.Fatalf("baseline = %v reason=%q, want day-three current-day baseline", agg.FirstEventTime, agg.QualityReason) + if !agg.FirstEventTime.Equal(dayOneTS) || agg.QualityReason != stats.QualityReasonHistorical { + t.Fatalf("baseline = %v reason=%q, want day-one historical baseline", agg.FirstEventTime, agg.QualityReason) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatalf("sql expectations: %v", err) @@ -602,10 +602,10 @@ func TestAddRealtimeLocationFallbackReusesAggregateHistoryAcrossEmptyDays(t *tes if agg == nil { t.Fatal("missing realtime-location fallback aggregate") } - if agg.FirstKM != 120 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayThreeTS) { - t.Fatalf("fallback range = %v@%v -> %v, want 120@day-three -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM) + if agg.FirstKM != 100 || agg.LatestKM != 120 || !agg.FirstEventTime.Equal(dayOneTS) { + t.Fatalf("fallback range = %v@%v -> %v, want 100@day-one -> 120", agg.FirstKM, agg.FirstEventTime, agg.LatestKM) } - if agg.QualityReason != "realtime_location_fallback_current_day_first_baseline" { + if agg.QualityReason != "realtime_location_fallback_historical_baseline" { t.Fatalf("quality reason = %q", agg.QualityReason) } if err := mysqlMock.ExpectationsWereMet(); err != nil { diff --git a/go/vehicle-gateway/internal/stats/daily_metric.go b/go/vehicle-gateway/internal/stats/daily_metric.go index 3202ee59..151bf385 100644 --- a/go/vehicle-gateway/internal/stats/daily_metric.go +++ b/go/vehicle-gateway/internal/stats/daily_metric.go @@ -924,9 +924,7 @@ func (w *Writer) applyRealtimeBaseline(ctx context.Context, candidate *SourceMil return nil } if !IsUsableDailyMileageBoundary(candidate.StatDate, baseline.LatestEventTime, w.loc) { - // A vehicle recovering after one or more empty natural days must start a - // fresh day boundary. Otherwise the complete offline-period odometer - // increase is incorrectly assigned to the recovery day. + // A future/invalid boundary cannot be used for this natural day. candidate.FirstTotalKM = candidate.LatestTotalKM candidate.FirstEventTime = candidate.LatestEventTime candidate.DailyKM = 0 diff --git a/go/vehicle-gateway/internal/stats/daily_metric_test.go b/go/vehicle-gateway/internal/stats/daily_metric_test.go index 9aa4557e..e1813c02 100644 --- a/go/vehicle-gateway/internal/stats/daily_metric_test.go +++ b/go/vehicle-gateway/internal/stats/daily_metric_test.go @@ -1453,7 +1453,7 @@ func TestWriterAppendUsesPreviousSourceBaselineForRealtimeCandidate(t *testing.T } } -func TestWriterAppendStartsCurrentDayBoundaryAfterOfflineGap(t *testing.T) { +func TestWriterAppendUsesHistoricalBoundaryAfterOfflineGap(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) @@ -1494,18 +1494,18 @@ func TestWriterAppendStartsCurrentDayBoundaryAfterOfflineGap(t *testing.T) { "", "LMRKH9AC2R1004087", "", + float64(120672.0), float64(120788.0), - float64(120788.0), - float64(0), + float64(116), float64(0), int64(1), int64(0), - currentTime, + historicalTime, currentTime, false, false, QualityOK, - QualityReasonCurrentDayFirst, + QualityReasonHistorical, ). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(`INSERT INTO vehicle_daily_mileage`). @@ -1711,7 +1711,7 @@ func TestWriterApplyRealtimeBaselineRejectsNegativeDeltaEvenWithLowerCurrentDayS } } -func TestWriterApplyRealtimeBaselineIgnoresPlausibleMultiDayFallbackDelta(t *testing.T) { +func TestWriterApplyRealtimeBaselineIncludesPlausibleMultiDayFallbackDelta(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) @@ -1740,14 +1740,14 @@ func TestWriterApplyRealtimeBaselineIgnoresPlausibleMultiDayFallbackDelta(t *tes if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { t.Fatalf("applyRealtimeBaseline() error = %v", err) } - if candidate.QualityStatus != QualityOK || candidate.QualityReason != QualityReasonCurrentDayFirst { + if candidate.QualityStatus != QualityOK || candidate.QualityReason != QualityReasonHistorical { t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) } - if candidate.DailyKM != 0 { - t.Fatalf("daily km = %v, want current-day first baseline after offline gap", candidate.DailyKM) + if math.Abs(candidate.DailyKM-7833.5) > 0.000001 { + t.Fatalf("daily km = %v, want historical cumulative difference after offline gap", candidate.DailyKM) } - if candidate.FirstTotalKM != candidate.LatestTotalKM || !candidate.FirstEventTime.Equal(currentTime) { - t.Fatalf("current-day boundary not retained: %#v", candidate) + if candidate.FirstTotalKM != 8832.1 || !candidate.FirstEventTime.Equal(baselineTime) { + t.Fatalf("historical boundary not retained: %#v", candidate) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatalf("sql expectations: %v", err) @@ -1798,7 +1798,7 @@ func TestWriterApplyRealtimeBaselineRejectsProtocolOdometerRollbackAfterSourceCh } } -func TestWriterApplyRealtimeBaselineIgnoresHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) { +func TestWriterApplyRealtimeBaselineRejectsHistoricalBaselineJumpAcrossOfflineGap(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("sqlmock.New() error = %v", err) @@ -1827,14 +1827,14 @@ func TestWriterApplyRealtimeBaselineIgnoresHistoricalBaselineJumpAcrossOfflineGa if err := writer.applyRealtimeBaseline(context.Background(), &candidate); err != nil { t.Fatalf("applyRealtimeBaseline() error = %v", err) } - if candidate.QualityStatus != QualityOK || candidate.QualityReason != QualityReasonCurrentDayFirst { + if candidate.QualityStatus != QualityInvalidDelta || candidate.QualityReason != "outside_daily_range" { t.Fatalf("quality = %s/%s", candidate.QualityStatus, candidate.QualityReason) } - if candidate.DailyKM != 0 { - t.Fatalf("daily km = %v, want current-day first baseline", candidate.DailyKM) + if math.Abs(candidate.DailyKM-30000) > 0.000001 { + t.Fatalf("daily km = %v, want historical cumulative difference", candidate.DailyKM) } - if candidate.FirstTotalKM != candidate.LatestTotalKM || !candidate.FirstEventTime.Equal(currentTime) { - t.Fatalf("current-day boundary not retained: %#v", candidate) + if candidate.FirstTotalKM != 10009.7 || !candidate.FirstEventTime.Equal(baselineTime) { + t.Fatalf("historical boundary not retained: %#v", candidate) } if err := mock.ExpectationsWereMet(); err != nil { t.Fatalf("sql expectations: %v", err) diff --git a/go/vehicle-gateway/internal/stats/source_mileage.go b/go/vehicle-gateway/internal/stats/source_mileage.go index 381421f9..61d6d98d 100644 --- a/go/vehicle-gateway/internal/stats/source_mileage.go +++ b/go/vehicle-gateway/internal/stats/source_mileage.go @@ -137,10 +137,8 @@ func DailyMileageFromDayBoundary(previousBaselineKM float64, currentDayLatestKM return currentDayLatestKM - previousBaselineKM } -// IsUsableDailyMileageBoundary reports whether a persisted baseline belongs to -// the current natural day or its immediately preceding natural day. A baseline -// from an older day represents an offline gap; using it would attribute the -// whole gap to the day on which the vehicle came back online. +// IsUsableDailyMileageBoundary accepts older same-source odometers. +// Offline increments belong to the recovery day so cumulative totals reconcile. func IsUsableDailyMileageBoundary(statDate string, baselineTime time.Time, loc *time.Location) bool { if strings.TrimSpace(statDate) == "" || baselineTime.IsZero() { return false @@ -154,7 +152,7 @@ func IsUsableDailyMileageBoundary(statDate string, baselineTime time.Time, loc * } baseline := baselineTime.In(loc) baselineDay := time.Date(baseline.Year(), baseline.Month(), baseline.Day(), 0, 0, 0, 0, loc) - return !baselineDay.Before(day.AddDate(0, 0, -1)) && !baselineDay.After(day) + return !baselineDay.After(day) } func NormalizeDailyMileageDeltaForWindow(deltaKM float64, firstEventTime time.Time, latestEventTime time.Time) (float64, bool, string) { @@ -754,7 +752,9 @@ const selectedSourceIdentitySampleCountSQL = `( // from the daily-mileage projection. GPS coordinate accumulation can be the // best evidence for distance travelled during the day, but it must never // replace a terminal-reported odometer as the day-end cumulative mileage. -const projectDayEndTotalMileageSQL = `COALESCE(( +const projectDayEndTotalMileageSQL = `CASE + WHEN COALESCE(s.quality_reason, '') <> '` + QualityReasonGPSCoordinate + `' THEN s.latest_total_mileage_km + ELSE COALESCE(( SELECT total_candidate.latest_total_mileage_km FROM vehicle_daily_mileage_source total_candidate WHERE total_candidate.vin = s.vin @@ -771,7 +771,7 @@ const projectDayEndTotalMileageSQL = `COALESCE(( total_candidate.sample_count DESC, total_candidate.source_key ASC LIMIT 1 -), s.latest_total_mileage_km)` +), s.latest_total_mileage_km) END` const projectDailyMileageSQL = ` INSERT INTO vehicle_daily_mileage diff --git a/go/vehicle-gateway/internal/stats/source_mileage_test.go b/go/vehicle-gateway/internal/stats/source_mileage_test.go index e0758315..95c42b49 100644 --- a/go/vehicle-gateway/internal/stats/source_mileage_test.go +++ b/go/vehicle-gateway/internal/stats/source_mileage_test.go @@ -279,7 +279,7 @@ func TestDailyMileageFromDayBoundaryUsesCurrentMinusPrevious(t *testing.T) { } } -func TestIsUsableDailyMileageBoundaryRejectsOlderOfflineGap(t *testing.T) { +func TestIsUsableDailyMileageBoundaryAcceptsOlderOfflineGap(t *testing.T) { loc := time.FixedZone("Asia/Shanghai", 8*3600) for _, test := range []struct { name string @@ -288,7 +288,7 @@ func TestIsUsableDailyMileageBoundaryRejectsOlderOfflineGap(t *testing.T) { }{ {name: "current day cache", baseline: time.Date(2026, 8, 4, 19, 1, 26, 0, loc), want: true}, {name: "previous natural day", baseline: time.Date(2026, 8, 3, 23, 59, 59, 0, loc), want: true}, - {name: "older offline gap", baseline: time.Date(2026, 5, 21, 23, 13, 2, 0, loc), want: false}, + {name: "older offline gap", baseline: time.Date(2026, 5, 21, 23, 13, 2, 0, loc), want: true}, {name: "future boundary", baseline: time.Date(2026, 8, 5, 0, 0, 0, 0, loc), want: false}, } { t.Run(test.name, func(t *testing.T) { diff --git a/reports/mileage-reconciliation-20260916/backup.py b/reports/mileage-reconciliation-20260916/backup.py new file mode 100644 index 00000000..2d6d8c36 --- /dev/null +++ b/reports/mileage-reconciliation-20260916/backup.py @@ -0,0 +1,49 @@ +"""Consistent logical backup of the complete current MySQL database; no DB writes.""" +import gzip, hashlib, json, os, pathlib, time +import pymysql +from db_access import connect +os.umask(0o077) +root=pathlib.Path('/opt/lingniu-go-native/backups/mileage-reconciliation-20260916') +root.mkdir(parents=True,exist_ok=True) +path=root/'database.sql.gz' +if path.exists(): raise RuntimeError('Backup already exists; do not overwrite') +c=connect(); manifest={'started_at':time.strftime('%Y-%m-%dT%H:%M:%S%z'),'tables':{}} +with c.cursor() as q: + q.execute('SELECT DATABASE() AS db'); db=q.fetchone()['db'];manifest['database']=db + q.execute('SHOW FULL TABLES'); entries=q.fetchall() + tables=[list(r.values())[0] for r in entries if list(r.values())[1]=='BASE TABLE'] + q.execute('SHOW TRIGGERS'); triggers=q.fetchall() + q.execute('START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY') + def literal(v): + if isinstance(v,bytes): return '0x'+v.hex() + return c.escape(v) + with gzip.open(str(path)+'.partial','wt',encoding='utf-8',compresslevel=3) as f: + f.write('SET NAMES utf8mb4;\nSET FOREIGN_KEY_CHECKS=0;\n') + for table in tables: + q.execute('SHOW CREATE TABLE `'+table+'`');ddl=list(q.fetchone().values())[1] + f.write('DROP TABLE IF EXISTS `'+table+'`;\n'+ddl+';\n') + q.execute('SHOW COLUMNS FROM `'+table+'`');cols=[r['Field'] for r in q.fetchall() if 'GENERATED' not in r['Extra']] + names=','.join('`'+v+'`' for v in cols);count=0 + with c.cursor(pymysql.cursors.SSCursor) as stream: + stream.execute('SELECT '+names+' FROM `'+table+'`') + while True: + rows=stream.fetchmany(500) + if not rows:break + f.write('INSERT INTO `'+table+'` ('+names+') VALUES\n'+',\n'.join('('+','.join(literal(v) for v in row)+')' for row in rows)+';\n') + count+=len(rows) + manifest['tables'][table]=count + print(json.dumps({'table':table,'rows':count}),flush=True) + for row in entries: + if list(row.values())[1]=='VIEW': + name=list(row.values())[0];q.execute('SHOW CREATE VIEW `'+name+'`');ddl=q.fetchone()['Create View'];f.write(ddl+';\n') + for trigger in triggers: + q.execute('SHOW CREATE TRIGGER `'+trigger['Trigger']+'`');ddl=q.fetchone()['SQL Original Statement'];f.write('DELIMITER ;;\n'+ddl+';;\nDELIMITER ;\n') + f.write('SET FOREIGN_KEY_CHECKS=1;\n') +c.rollback();c.close() +os.rename(str(path)+'.partial',path) +h=hashlib.sha256() +with path.open('rb') as f: + for block in iter(lambda:f.read(1024*1024),b''):h.update(block) +manifest.update(sha256=h.hexdigest(),bytes=path.stat().st_size,finished_at=time.strftime('%Y-%m-%dT%H:%M:%S%z')) +(root/'manifest.json').write_text(json.dumps(manifest,indent=2)) +print(json.dumps({'complete':True,'bytes':manifest['bytes'],'sha256':manifest['sha256']}),flush=True) diff --git a/reports/mileage-reconciliation-20260916/db_access.py b/reports/mileage-reconciliation-20260916/db_access.py new file mode 100644 index 00000000..dfb819c9 --- /dev/null +++ b/reports/mileage-reconciliation-20260916/db_access.py @@ -0,0 +1,17 @@ +import pymysql,pathlib,re,json + +def connect(): + env={} + for name in ['base.env','stat-writer.env']: + for line in pathlib.Path('/opt/lingniu-go-native/env/'+name).read_text().splitlines(): + if '=' in line and not line.startswith('#'): + k,v=line.split('=',1);env[k]=v.strip().strip(chr(34)).strip(chr(39)) + m=re.fullmatch(r'([^:]+):(.*?)@tcp\(([^:]+):(\d+)\)/([^?]+).*',env['MYSQL_DSN']) + return pymysql.connect(user=m[1],password=m[2],host=m[3],port=int(m[4]),database=m[5],charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor,autocommit=False) +if __name__=='__main__': + c=connect() + with c.cursor() as q: + q.execute('START TRANSACTION READ ONLY') + for sql in ["SELECT DATABASE() AS db, VERSION() AS version", "SELECT table_name,engine,table_rows,ROUND((data_length+index_length)/1024/1024,1) AS mb FROM information_schema.tables WHERE table_schema=DATABASE() ORDER BY data_length DESC", "SELECT protocol,source_ip,COUNT(*) AS n,COUNT(DISTINCT vin) AS vins FROM vehicle_daily_mileage_source WHERE source_ip IN ('legacy-mysql.lingniu-prod','manual-lingniu-prod-day-mileage') GROUP BY protocol,source_ip"]: + q.execute(sql);print(json.dumps(q.fetchall(),default=str)) + c.rollback();c.close() diff --git a/reports/mileage-reconciliation-20260916/repair.py b/reports/mileage-reconciliation-20260916/repair.py new file mode 100644 index 00000000..9f2ab93c --- /dev/null +++ b/reports/mileage-reconciliation-20260916/repair.py @@ -0,0 +1,60 @@ +"""Repair verified mileage provenance and offline baselines. Dry-run by default.""" +import argparse,datetime,json,pathlib +from decimal import Decimal +from db_access import connect +p=argparse.ArgumentParser();p.add_argument('--apply',action='store_true');args=p.parse_args() +root=pathlib.Path('/opt/lingniu-go-native/backups/mileage-reconciliation-20260916') +legacy="('legacy-mysql.lingniu-prod','manual-lingniu-prod-day-mileage')" +c=connect();report={} +with c.cursor() as q: + q.execute('START TRANSACTION READ ONLY') + q.execute("SELECT COUNT(*) AS n,COUNT(DISTINCT vin) AS vehicles FROM vehicle_daily_mileage_source WHERE protocol='GB32960' AND source_ip IN "+legacy);report['legacy']=q.fetchone() + q.execute("SELECT COUNT(*) AS n FROM vehicle_daily_mileage m JOIN vehicle_data_source d ON d.id=m.source_id WHERE m.protocol='GB32960' AND d.source_ip IN "+legacy);report['legacy_projections']=q.fetchone() + q.execute("""SELECT s.vin,s.stat_date,s.protocol,s.source_key,s.latest_total_mileage_km,s.first_total_mileage_km,s.daily_mileage_km,s.first_event_time,s.latest_event_time, + p.latest_total_mileage_km AS previous_total,p.latest_event_time AS previous_time + FROM vehicle_daily_mileage_source s + JOIN vehicle_daily_mileage_source p ON p.vin=s.vin AND p.protocol=s.protocol AND p.source_key=s.source_key + AND p.stat_date=(SELECT MAX(b.stat_date) FROM vehicle_daily_mileage_source b + WHERE b.vin=s.vin AND b.protocol=s.protocol AND b.source_key=s.source_key AND b.stat_date0 + AND b.latest_event_time>=TIMESTAMP(b.stat_date) AND b.latest_event_time0 AND s.first_event_time>=TIMESTAMP(s.stat_date) + AND s.source_ip NOT IN """+legacy+" ORDER BY s.vin,s.protocol,s.source_key,s.stat_date") + changes=q.fetchall();report['baseline_candidates']=len(changes) + q.execute("SELECT id FROM vehicle_data_source WHERE source_ip IN "+legacy);legacy_ids=[r['id'] for r in q.fetchall()] + c.rollback() + if args.apply: + assert (root/'manifest.json').is_file() and (root/'database.sql.gz').is_file(),'Complete backup required' + (root/'baseline-before.json').write_text(json.dumps(changes,default=str,ensure_ascii=False)) + q.execute('START TRANSACTION') + q.execute("UPDATE vehicle_daily_mileage_source SET quality_status='INVALID_DELTA',quality_reason='UNVERIFIED_LEGACY_PROTOCOL',is_selected=0 WHERE protocol='GB32960' AND source_ip IN "+legacy) + report['quarantined_source_rows']=q.rowcount + q.execute("UPDATE vehicle_data_source SET enabled=0,remark='Protocol provenance unverified; quarantined 2026-09-16; original records retained' WHERE protocol='GB32960' AND source_ip IN "+legacy) + q.execute("DELETE m FROM vehicle_daily_mileage m JOIN vehicle_data_source d ON d.id=m.source_id WHERE m.protocol='GB32960' AND d.source_ip IN "+legacy) + report['removed_mislabelled_projections']=q.rowcount + affected=set() + for s in changes: + delta=s['latest_total_mileage_km']-s['previous_total'] + days=max(1,(s['latest_event_time'].date()-s['previous_time'].date()).days) + status='OK';reason='historical_source_baseline' + if Decimal('-1')<=delta<0:delta=Decimal(0);reason='negative_jitter_clamped' + elif delta<0:status='INVALID_DELTA';reason='TOTAL_MILEAGE_ROLLBACK' + elif delta>Decimal(2500*days):status='INVALID_DELTA';reason='outside_daily_range' + q.execute("""UPDATE vehicle_daily_mileage_source SET first_total_mileage_km=%s,first_event_time=%s,daily_mileage_km=%s, + quality_status=%s,quality_reason=%s WHERE vin=%s AND stat_date=%s AND protocol=%s AND source_key=%s + AND first_total_mileage_km <=> %s AND daily_mileage_km=%s""",(s['previous_total'],s['previous_time'],delta,status,reason,s['vin'],s['stat_date'],s['protocol'],s['source_key'],s['first_total_mileage_km'],s['daily_mileage_km'])) + if q.rowcount:affected.add((s['vin'],s['stat_date'],s['protocol'])) + # Restore valid non-import GB32960 projections that were hidden by migrated rows. + q.execute("""SELECT DISTINCT s.vin,s.stat_date,s.protocol FROM vehicle_daily_mileage_source s + JOIN vehicle_daily_mileage_source bad ON bad.vin=s.vin AND bad.stat_date=s.stat_date AND bad.protocol=s.protocol + WHERE bad.source_ip IN """+legacy+" AND s.source_ip NOT IN "+legacy+" AND s.quality_status='OK' AND s.latest_total_mileage_km IS NOT NULL") + affected.update((r['vin'],r['stat_date'],r['protocol']) for r in q.fetchall()) + report['reproject_days']=len(affected) + # The Go projection utility is run before resuming writers, using canonical SQL. + (root/'reproject.json').write_text(json.dumps([{'vin':v,'date':str(d),'protocol':p} for v,d,p in sorted(affected)])) + c.commit() + (root if args.apply else pathlib.Path('/tmp')).joinpath('repair-result.json' if args.apply else 'mileage-repair-dry-run.json').write_text(json.dumps(report,default=str,indent=2)) + print(json.dumps(report,default=str),flush=True) +c.close() diff --git a/reports/mileage-reconciliation-20260916/run_remote.py b/reports/mileage-reconciliation-20260916/run_remote.py new file mode 100644 index 00000000..575e113f --- /dev/null +++ b/reports/mileage-reconciliation-20260916/run_remote.py @@ -0,0 +1,7 @@ +"""Run a specified maintenance binary without exposing environment credentials.""" +import os,pathlib,subprocess,sys +for name in ['base.env','stat-writer.env']: + for line in pathlib.Path('/opt/lingniu-go-native/env/'+name).read_text().splitlines(): + if '=' in line and not line.startswith('#'): + k,v=line.split('=',1);os.environ[k]=v.strip().strip(chr(34)).strip(chr(39)) +sys.exit(subprocess.call(sys.argv[1:])) diff --git a/tmp/g7-mileage-eval-20260724/inspect_sample.mjs b/tmp/g7-mileage-eval-20260724/inspect_sample.mjs new file mode 100644 index 00000000..3769466e --- /dev/null +++ b/tmp/g7-mileage-eval-20260724/inspect_sample.mjs @@ -0,0 +1,115 @@ +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const inputPath = + "/Users/lingniu/Downloads/智能管车_车辆里程日报-105652096.xlsx"; +const input = await FileBlob.load(inputPath); +const workbook = await SpreadsheetFile.importXlsx(input); + +const summary = await workbook.inspect({ + kind: "workbook,sheet,table", + maxChars: 12000, + tableMaxRows: 8, + tableMaxCols: 40, + tableMaxCellChars: 120, +}); + +const sheet = workbook.worksheets.getItemAt(0); +const usedRange = sheet.getUsedRange(true); +const rowCount = usedRange?.rowCount ?? 0; +const columnCount = usedRange?.columnCount ?? 0; +const values = usedRange?.values ?? []; +const headers = values[0] ?? []; +const dateStartIndex = 3; +const dateEndIndex = Math.max(dateStartIndex, headers.indexOf("运行时长")); +let positiveDailyCellCount = 0; +let nonZeroVehicleCount = 0; +let noteVehicleCount = 0; +let mismatchVehicleCount = 0; +let totalDailyKm = 0; +let totalColumnKm = 0; +const nonZeroSamples = []; +const noteSamples = []; +for (const row of values.slice(1)) { + const dailyValues = row + .slice(dateStartIndex, dateEndIndex) + .map(numberValue); + const positiveValues = dailyValues.filter((value) => value > 0); + const rowDailyKm = dailyValues.reduce((sum, value) => sum + value, 0); + const rowTotalKm = numberValue(row[2]); + const note = row[dateEndIndex + 1]; + positiveDailyCellCount += positiveValues.length; + totalDailyKm += rowDailyKm; + totalColumnKm += rowTotalKm; + if (positiveValues.length > 0) { + nonZeroVehicleCount += 1; + if (nonZeroSamples.length < 20) { + nonZeroSamples.push({ + plate: row[0], + organization: row[1], + totalKm: rowTotalKm, + dailyKm: rowDailyKm, + positiveDayCount: positiveValues.length, + }); + } + } + if (note !== null && note !== undefined && String(note).trim() !== "") { + noteVehicleCount += 1; + if (noteSamples.length < 20) { + noteSamples.push({ plate: row[0], note }); + } + } + if (Math.abs(rowTotalKm - rowDailyKm) > 0.011) { + mismatchVehicleCount += 1; + } +} +const inspectRange = rowCount > 0 && columnCount > 0 + ? `A1:${columnName(Math.min(columnCount, 40))}${Math.min(rowCount, 15)}` + : "A1:A1"; +const preview = await workbook.inspect({ + kind: "table", + sheetId: sheet.name, + range: inspectRange, + include: "values,formulas", + maxChars: 25000, + tableMaxRows: 15, + tableMaxCols: 40, + tableMaxCellChars: 120, +}); + +console.log(JSON.stringify({ + inputPath, + sheetName: sheet.name, + rowCount, + columnCount, + dateColumnCount: Math.max(0, dateEndIndex - dateStartIndex), + positiveDailyCellCount, + nonZeroVehicleCount, + noteVehicleCount, + mismatchVehicleCount, + totalDailyKm: round(totalDailyKm), + totalColumnKm: round(totalColumnKm), + nonZeroSamples, + noteSamples, + summary: summary.ndjson, + preview: preview.ndjson, +}, null, 2)); + +function columnName(columnCount) { + let value = columnCount; + let name = ""; + while (value > 0) { + value -= 1; + name = String.fromCharCode(65 + (value % 26)) + name; + value = Math.floor(value / 26); + } + return name; +} + +function numberValue(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : 0; +} + +function round(value) { + return Math.round(value * 1000) / 1000; +} diff --git a/tmp/g7-mileage-history-20260724/build_all.mjs b/tmp/g7-mileage-history-20260724/build_all.mjs new file mode 100644 index 00000000..3ff063be --- /dev/null +++ b/tmp/g7-mileage-history-20260724/build_all.mjs @@ -0,0 +1,741 @@ +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import path from "node:path"; +import { once } from "node:events"; +import { + FileBlob, + SpreadsheetFile, + Workbook, +} from "@oai/artifact-tool"; + +const root = + "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest"; +const workDir = path.join(root, "outputs/g7-mileage-history-20260724"); +const dailyDir = path.join(workDir, "raw/daily"); +const statisticDir = path.join(workDir, "raw/statistic"); +const mergedDir = path.join(workDir, "merged"); +const outputXlsx = path.join( + mergedDir, + "G7车辆每日里程及里程统计_20220101-20260720.xlsx", +); +const importCsv = path.join( + mergedDir, + "G7车辆每日里程_20220101-20260720.csv", +); +const manifestPath = path.join(mergedDir, "manifest.json"); +const priorMappingCsv = path.join( + root, + "outputs/g7-mileage-import-20260716/import_rows.csv", +); + +await fs.mkdir(mergedDir, { recursive: true }); + +const months = buildMonths(); +const dailyFilesByTaskId = await sortedXlsxFiles(dailyDir); +// 2022-02 was the one-file validation task. The remaining formal queue then +// started with 2022-01 and continued from 2022-03, so swap the first two task IDs +// back into natural month order. +const dailyFiles = [ + dailyFilesByTaskId[1], + dailyFilesByTaskId[0], + ...dailyFilesByTaskId.slice(2), +]; +const statisticFiles = await sortedXlsxFiles(statisticDir); +assert(dailyFiles.length === months.length, `日报文件应为 ${months.length}，实际 ${dailyFiles.length}`); +assert( + statisticFiles.length === months.length, + `里程统计文件应为 ${months.length}，实际 ${statisticFiles.length}`, +); + +const vehicles = new Map(); +const dateOrder = []; +const dateSource = new Map(); +const monthlyDailyTotals = new Map(); +const dailyBatchManifest = []; +let dailyPositiveCells = 0; +let dailyZeroCells = 0; +let dailyTotalKm = 0; +let dailyTotalMismatchRows = 0; +let dailyCompletenessNoteRows = 0; + +for (let monthIndex = 0; monthIndex < months.length; monthIndex += 1) { + const month = months[monthIndex]; + const file = dailyFiles[monthIndex]; + const values = await readFirstSheetValues(file); + const headers = values[0] ?? []; + const runtimeIndex = headers.indexOf("运行时长"); + assert(runtimeIndex > 3, `${path.basename(file)} 缺少运行时长列`); + const dateColumnCount = runtimeIndex - 3; + assert( + dateColumnCount === month.dayCount, + `${path.basename(file)} 日期列应为 ${month.dayCount}，实际 ${dateColumnCount}`, + ); + + const monthDates = []; + for (let day = 1; day <= month.dayCount; day += 1) { + const date = `${month.year}-${pad(month.month)}-${pad(day)}`; + monthDates.push(date); + dateOrder.push(date); + dateSource.set(date, path.basename(file)); + } + + let batchTotal = 0; + let batchPositive = 0; + let batchZero = 0; + let batchMismatch = 0; + let batchNotes = 0; + const seenPlates = new Set(); + for (let rowIndex = 1; rowIndex < values.length; rowIndex += 1) { + const row = values[rowIndex] ?? []; + const plate = cleanText(row[0]); + if (!plate) continue; + assert(!seenPlates.has(plate), `${path.basename(file)} 存在重复车牌 ${plate}`); + seenPlates.add(plate); + const organization = cleanText(row[1]); + const statedTotal = numberValue(row[2]); + const dailyValues = row.slice(3, runtimeIndex).map(numberValue); + const calculatedTotal = dailyValues.reduce((sum, value) => sum + value, 0); + if (Math.abs(statedTotal - calculatedTotal) > 0.011) { + dailyTotalMismatchRows += 1; + batchMismatch += 1; + } + const note = cleanText(row[runtimeIndex + 1]); + if (note) { + dailyCompletenessNoteRows += 1; + batchNotes += 1; + } + + let vehicle = vehicles.get(plate); + if (!vehicle) { + vehicle = { + plate, + organization, + values: new Map(), + }; + vehicles.set(plate, vehicle); + } else if (!vehicle.organization && organization) { + vehicle.organization = organization; + } + + for (let dayIndex = 0; dayIndex < dailyValues.length; dayIndex += 1) { + const value = dailyValues[dayIndex]; + const date = monthDates[dayIndex]; + vehicle.values.set(date, value); + batchTotal += value; + dailyTotalKm += value; + if (value > 0) { + batchPositive += 1; + dailyPositiveCells += 1; + } else { + batchZero += 1; + dailyZeroCells += 1; + } + } + } + monthlyDailyTotals.set(month.key, round(batchTotal)); + dailyBatchManifest.push({ + type: "车辆里程日报", + month: month.key, + date_from: month.dateFrom, + date_to: month.dateTo, + file: path.basename(file), + vehicle_rows: seenPlates.size, + date_columns: month.dayCount, + positive_cells: batchPositive, + zero_cells: batchZero, + total_km: round(batchTotal), + total_mismatch_rows: batchMismatch, + completeness_note_rows: batchNotes, + }); +} + +assert(dateOrder.length === 1662, `日期列应为 1662，实际 ${dateOrder.length}`); +assert(new Set(dateOrder).size === dateOrder.length, "日报日期列存在重复"); +assert(dateOrder[0] === "2022-01-01", `起始日期异常：${dateOrder[0]}`); +assert(dateOrder.at(-1) === "2026-07-20", `结束日期异常：${dateOrder.at(-1)}`); + +const statisticRows = []; +const monthlyStatisticTotals = new Map(); +const statisticBatchManifest = []; +for (let monthIndex = 0; monthIndex < months.length; monthIndex += 1) { + const month = months[monthIndex]; + const file = statisticFiles[monthIndex]; + const values = await readFirstSheetValues(file); + const headers = (values[0] ?? []).map(cleanText); + const plateIndex = findHeader(headers, ["车牌号"]); + const orgIndex = findHeader(headers, ["所属机构", "机构"]); + const sourceOrgIndex = findHeader(headers, ["来源机构"]); + const lengthIndex = findHeader(headers, ["车长(米)", "车长"]); + const boxIndex = findHeader(headers, ["厢型"]); + const runtimeIndex = findHeader(headers, ["运行时长"]); + const mileageIndex = findHeader(headers, ["行驶里程(KM)", "行驶里程"]); + assert(plateIndex >= 0, `${path.basename(file)} 缺少车牌号`); + assert(mileageIndex >= 0, `${path.basename(file)} 缺少行驶里程`); + + let batchTotal = 0; + let vehicleRows = 0; + for (let rowIndex = 1; rowIndex < values.length; rowIndex += 1) { + const row = values[rowIndex] ?? []; + const plate = cleanText(row[plateIndex]); + if (!plate) continue; + const mileage = numberValue(row[mileageIndex]); + batchTotal += mileage; + vehicleRows += 1; + statisticRows.push([ + month.key, + plate, + orgIndex >= 0 ? cleanText(row[orgIndex]) : "", + sourceOrgIndex >= 0 ? cleanText(row[sourceOrgIndex]) : "", + lengthIndex >= 0 ? row[lengthIndex] ?? "" : "", + boxIndex >= 0 ? cleanText(row[boxIndex]) : "", + runtimeIndex >= 0 ? cleanText(row[runtimeIndex]) : "", + mileage, + path.basename(file), + ]); + } + monthlyStatisticTotals.set(month.key, round(batchTotal)); + statisticBatchManifest.push({ + type: "车辆里程统计", + month: month.key, + date_from: month.dateFrom, + date_to: month.dateTo, + file: path.basename(file), + vehicle_rows: vehicleRows, + total_km: round(batchTotal), + }); +} + +const mapping = await readPriorMapping(priorMappingCsv); +const sortedVehicles = [...vehicles.values()].sort((a, b) => + a.plate.localeCompare(b.plate, "zh-CN"), +); +const mappedVehicles = sortedVehicles.filter((vehicle) => mapping.has(vehicle.plate)); +const unmappedVehicles = sortedVehicles.filter((vehicle) => !mapping.has(vehicle.plate)); + +await writeImportCsv(importCsv, sortedVehicles, dateOrder); + +const workbook = Workbook.create(); +const summarySheet = workbook.worksheets.add("汇总"); +const positiveDailySheet = workbook.worksheets.add("每日里程非零"); +const statisticSheet = workbook.worksheets.add("G7里程统计"); +const mappingSheet = workbook.worksheets.add("车辆映射"); +const batchSheet = workbook.worksheets.add("导出批次"); + +buildSummarySheet(summarySheet); +await buildPositiveDailySheet(positiveDailySheet); +await buildStatisticSheet(statisticSheet); +await buildMappingSheet(mappingSheet); +await buildBatchSheet(batchSheet); + +const summaryInspect = await workbook.inspect({ + kind: "table", + sheetId: "汇总", + range: "A1:F25", + include: "values,formulas", + maxChars: 12000, + tableMaxRows: 25, + tableMaxCols: 6, +}); +const dailyInspect = await workbook.inspect({ + kind: "table", + sheetId: "每日里程非零", + range: "A1:D12", + include: "values,formulas", + maxChars: 12000, + tableMaxRows: 12, + tableMaxCols: 4, +}); +const formulaErrors = await workbook.inspect({ + kind: "match", + searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", + options: { useRegex: true, maxResults: 300 }, + summary: "final formula error scan", +}); +assert(!formulaErrors.ndjson.includes('"match"'), "工作簿存在公式错误"); + +const summaryPreview = await workbook.render({ + sheetName: "汇总", + range: "A1:F25", + scale: 1.5, + format: "png", +}); +await fs.writeFile( + path.join(mergedDir, "summary-preview.png"), + new Uint8Array(await summaryPreview.arrayBuffer()), +); +const dailyPreview = await workbook.render({ + sheetName: "每日里程非零", + range: "A1:D12", + scale: 1.2, + format: "png", +}); +await fs.writeFile( + path.join(mergedDir, "daily-preview.png"), + new Uint8Array(await dailyPreview.arrayBuffer()), +); + +try { + const output = await SpreadsheetFile.exportXlsx(workbook); + await output.save(outputXlsx); +} catch (error) { + console.error( + JSON.stringify( + { + stage: "export_xlsx", + name: error?.name, + message: error?.message, + stack: String(error?.stack ?? "") + .split("\n") + .filter((line) => !line.includes("artifact_tool.mjs:3121")) + .slice(0, 12), + }, + null, + 2, + ), + ); + process.exit(1); +} + +const monthlyReconciliation = months.map((month) => { + const dailyKm = monthlyDailyTotals.get(month.key) ?? 0; + const statisticKm = monthlyStatisticTotals.get(month.key) ?? 0; + return { + month: month.key, + daily_km: dailyKm, + statistic_km: statisticKm, + difference_km: round(dailyKm - statisticKm), + }; +}); +const manifest = { + generated_at: new Date().toISOString(), + range: { + date_from: dateOrder[0], + date_to: dateOrder.at(-1), + days: dateOrder.length, + months: months.length, + }, + files: { + daily_raw: dailyFiles.map((file) => path.basename(file)), + statistic_raw: statisticFiles.map((file) => path.basename(file)), + workbook: path.basename(outputXlsx), + import_csv: path.basename(importCsv), + }, + vehicle_count: sortedVehicles.length, + mapped_vehicle_count: mappedVehicles.length, + unmapped_vehicle_count: unmappedVehicles.length, + vehicle_day_rows: sortedVehicles.length * dateOrder.length, + mapped_vehicle_day_rows: mappedVehicles.length * dateOrder.length, + positive_cells: dailyPositiveCells, + zero_cells: dailyZeroCells, + daily_total_km: round(dailyTotalKm), + daily_total_mismatch_rows: dailyTotalMismatchRows, + completeness_note_rows: dailyCompletenessNoteRows, + monthly_reconciliation: monthlyReconciliation, + daily_batches: dailyBatchManifest, + statistic_batches: statisticBatchManifest, + verification: { + summary_inspect: summaryInspect.ndjson, + daily_inspect: dailyInspect.ndjson, + formula_errors: formulaErrors.ndjson, + }, +}; +await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + +console.log( + JSON.stringify( + { + outputXlsx, + importCsv, + manifestPath, + vehicleCount: sortedVehicles.length, + mappedVehicleCount: mappedVehicles.length, + unmappedVehicleCount: unmappedVehicles.length, + dateCount: dateOrder.length, + vehicleDayRows: sortedVehicles.length * dateOrder.length, + mappedVehicleDayRows: mappedVehicles.length * dateOrder.length, + positiveCells: dailyPositiveCells, + zeroCells: dailyZeroCells, + totalKm: round(dailyTotalKm), + monthlyDifferenceCount: monthlyReconciliation.filter( + (row) => Math.abs(row.difference_km) > 0.02, + ).length, + }, + null, + 2, + ), +); + +function buildSummarySheet(sheet) { + sheet.showGridLines = false; + sheet.getRange("A1:F1").merge(); + sheet.getRange("A1").values = [["G7 车辆里程全量导出汇总"]]; + sheet.getRange("A1:F1").format = { + fill: "#0F766E", + font: { bold: true, color: "#FFFFFF", size: 16 }, + horizontalAlignment: "center", + verticalAlignment: "center", + }; + sheet.getRange("A1:F1").format.rowHeight = 30; + + const metrics = [ + ["指标", "值", "说明"], + ["统计起始日期", new Date(`${dateOrder[0]}T00:00:00+08:00`), "G7 日报"], + ["统计结束日期", new Date(`${dateOrder.at(-1)}T00:00:00+08:00`), "含当日"], + ["自然日数", dateOrder.length, "连续日期"], + ["日报导出批次", dailyFiles.length, "每月 1 份"], + ["里程统计导出批次", statisticFiles.length, "每月 1 份"], + ["G7 车辆数", sortedVehicles.length, "当前导出车辆列表"], + ["已映射 VIN 车辆数", mappedVehicles.length, "沿用现有车辆映射"], + ["未映射车辆数", unmappedVehicles.length, "不进入正式导入"], + ["映射覆盖率", null, "已映射 / G7 车辆数"], + ["车辆-日期单元格", sortedVehicles.length * dateOrder.length, "包含 0 km"], + ["可导入车辆-日期行", mappedVehicles.length * dateOrder.length, "包含 0 km"], + ["正里程单元格", dailyPositiveCells, "> 0 km"], + ["0 km 单元格", dailyZeroCells, "按用户确认保留"], + ["日报累计里程(km)", round(dailyTotalKm), "55 份日报求和"], + ["日报合计不一致车辆月", dailyTotalMismatchRows, "日报“行驶里程”与逐日求和"], + ["完整度说明非空车辆月", dailyCompletenessNoteRows, "来自 G7 日报"], + ]; + sheet.getRangeByIndexes(2, 0, metrics.length, 3).values = metrics; + sheet.getRange("A3:C3").format = headerFormat(); + sheet.getRange("B4:B5").format.numberFormat = "yyyy-mm-dd"; + sheet.getRange("B6:B19").format.numberFormat = "#,##0.00"; + sheet.getRange("B12").formulas = [["=IFERROR(B10/B9,0)"]]; + sheet.getRange("B12").format.numberFormat = "0.0%"; + sheet.getRange("A4:A19").format.font = { bold: true, color: "#134E4A" }; + sheet.getRange("A3:C19").format.borders = { + preset: "inside", + style: "thin", + color: "#D1D5DB", + }; + + sheet.getRange("E3:H3").values = [["月份", "日报合计(km)", "里程统计合计(km)", "差异(km)"]]; + sheet.getRange("E3:H3").format = headerFormat(); + const monthlyRows = months.map((month) => [ + month.key, + monthlyDailyTotals.get(month.key) ?? 0, + monthlyStatisticTotals.get(month.key) ?? 0, + null, + ]); + sheet.getRangeByIndexes(3, 4, monthlyRows.length, 4).values = monthlyRows; + sheet.getRange("H4").formulas = [["=F4-G4"]]; + sheet.getRange(`H4:H${3 + monthlyRows.length}`).fillDown(); + sheet.getRange(`F4:H${3 + monthlyRows.length}`).format.numberFormat = "#,##0.00"; + sheet.getRange(`E3:H${3 + monthlyRows.length}`).format.borders = { + preset: "inside", + style: "thin", + color: "#E5E7EB", + }; + + sheet.freezePanes.freezeRows(3); + sheet.getRange("A21:C23").merge(); + sheet.getRange("A21").values = [[ + "Excel 的“每日里程非零”工作表列出全部正里程记录；未出现的车辆×日期组合按用户确认均为 0 km。含全部 0 km 的逐行明细保存在同目录 CSV：G7车辆每日里程_20220101-20260720.csv。", + ]]; + sheet.getRange("A21:C23").format = { + fill: "#ECFDF5", + font: { color: "#065F46", italic: true }, + wrapText: true, + verticalAlignment: "center", + }; + sheet.getRange(`A1:H${3 + monthlyRows.length}`).format.font = { name: "Aptos", size: 10 }; + sheet.getRange(`A1:A${3 + monthlyRows.length}`).format.columnWidth = 24; + sheet.getRange(`B1:B${3 + monthlyRows.length}`).format.columnWidth = 18; + sheet.getRange(`C1:C${3 + monthlyRows.length}`).format.columnWidth = 30; + sheet.getRange(`D1:D${3 + monthlyRows.length}`).format.columnWidth = 3; + sheet.getRange(`E1:E${3 + monthlyRows.length}`).format.columnWidth = 12; + sheet.getRange(`F1:H${3 + monthlyRows.length}`).format.columnWidth = 18; +} + +async function buildPositiveDailySheet(sheet) { + sheet.showGridLines = false; + const headers = ["日期", "车牌号", "机构", "每日里程(km)"]; + sheet.getRange("A1:D1").values = [headers]; + sheet.getRange("A1:D1").format = headerFormat(); + const rows = []; + for (const vehicle of sortedVehicles) { + for (const date of dateOrder) { + const mileage = vehicle.values.get(date) ?? 0; + if (mileage <= 0) continue; + rows.push([ + new Date(`${date}T00:00:00+08:00`), + vehicle.plate, + vehicle.organization, + mileage, + ]); + } + } + const chunkSize = 1000; + for (let start = 0; start < rows.length; start += chunkSize) { + const chunk = rows.slice(start, start + chunkSize); + sheet.getRangeByIndexes(start + 1, 0, chunk.length, headers.length).values = chunk; + } + sheet.getRange(`A2:A${rows.length + 1}`).format.numberFormat = "yyyy-mm-dd"; + sheet.getRange(`D2:D${rows.length + 1}`).format.numberFormat = "#,##0.00"; + sheet.freezePanes.freezeRows(1); + [14, 16, 30, 18].forEach((width, index) => { + sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width; + }); +} + +async function buildStatisticSheet(sheet) { + sheet.showGridLines = false; + const headers = [ + "月份", + "车牌号", + "所属机构", + "来源机构", + "车长(米)", + "厢型", + "运行时长", + "行驶里程(KM)", + "来源文件", + ]; + sheet.getRange("A1:I1").values = [headers]; + sheet.getRange("A1:I1").format = headerFormat(); + const chunkSize = 1000; + for (let start = 0; start < statisticRows.length; start += chunkSize) { + const chunk = statisticRows.slice(start, start + chunkSize); + sheet.getRangeByIndexes(start + 1, 0, chunk.length, headers.length).values = chunk; + } + sheet.getRange(`E2:E${statisticRows.length + 1}`).format.numberFormat = "0.0"; + sheet.getRange(`H2:H${statisticRows.length + 1}`).format.numberFormat = "#,##0.00"; + sheet.freezePanes.freezeRows(1); + const widths = [12, 16, 28, 28, 12, 16, 18, 18, 34]; + widths.forEach((width, index) => { + sheet.getRangeByIndexes(0, index, statisticRows.length + 1, 1).format.columnWidth = width; + }); +} + +async function buildMappingSheet(sheet) { + sheet.showGridLines = false; + const headers = ["车牌号", "机构", "VIN", "终端号", "映射状态", "期间里程(km)", "正里程天数", "0 km天数"]; + sheet.getRange("A1:H1").values = [headers]; + sheet.getRange("A1:H1").format = headerFormat(); + const rows = sortedVehicles.map((vehicle) => { + const mapped = mapping.get(vehicle.plate); + const allValues = dateOrder.map((date) => vehicle.values.get(date) ?? 0); + const positiveDays = allValues.filter((value) => value > 0).length; + return [ + vehicle.plate, + vehicle.organization, + mapped?.vin ?? "", + mapped?.phone ?? "", + mapped ? "已映射" : "未映射", + round(allValues.reduce((sum, value) => sum + value, 0)), + positiveDays, + dateOrder.length - positiveDays, + ]; + }); + sheet.getRangeByIndexes(1, 0, rows.length, headers.length).values = rows; + sheet.getRange(`F2:F${rows.length + 1}`).format.numberFormat = "#,##0.00"; + sheet.getRange(`E2:E${rows.length + 1}`).conditionalFormats.add("containsText", { + text: "未映射", + format: { fill: "#FEE2E2", font: { color: "#991B1B", bold: true } }, + }); + sheet.freezePanes.freezeRows(1); + [16, 30, 22, 18, 14, 18, 14, 14].forEach((width, index) => { + sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width; + }); +} + +async function buildBatchSheet(sheet) { + sheet.showGridLines = false; + const headers = [ + "类型", + "月份", + "开始日期", + "结束日期", + "来源文件", + "车辆数", + "日期列数", + "正里程单元格", + "0 km 单元格", + "合计里程(km)", + ]; + sheet.getRange("A1:J1").values = [headers]; + sheet.getRange("A1:J1").format = headerFormat(); + const rows = [ + ...dailyBatchManifest.map((batch) => [ + batch.type, + batch.month, + new Date(`${batch.date_from}T00:00:00+08:00`), + new Date(`${batch.date_to}T00:00:00+08:00`), + batch.file, + batch.vehicle_rows, + batch.date_columns, + batch.positive_cells, + batch.zero_cells, + batch.total_km, + ]), + ...statisticBatchManifest.map((batch) => [ + batch.type, + batch.month, + new Date(`${batch.date_from}T00:00:00+08:00`), + new Date(`${batch.date_to}T00:00:00+08:00`), + batch.file, + batch.vehicle_rows, + "", + "", + "", + batch.total_km, + ]), + ]; + sheet.getRangeByIndexes(1, 0, rows.length, headers.length).values = rows; + sheet.getRange(`C2:D${rows.length + 1}`).format.numberFormat = "yyyy-mm-dd"; + sheet.getRange(`F2:J${rows.length + 1}`).format.numberFormat = "#,##0.00"; + sheet.freezePanes.freezeRows(1); + [18, 12, 14, 14, 36, 12, 12, 18, 18, 18].forEach((width, index) => { + sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width; + }); +} + +async function readFirstSheetValues(file) { + const input = await FileBlob.load(file); + const workbook = await SpreadsheetFile.importXlsx(input); + const sheet = workbook.worksheets.getItemAt(0); + const usedRange = sheet.getUsedRange(true); + return usedRange?.values ?? []; +} + +async function sortedXlsxFiles(directory) { + const names = (await fs.readdir(directory)) + .filter((name) => name.endsWith(".xlsx")) + .sort((a, b) => extractTaskId(a) - extractTaskId(b)); + return names.map((name) => path.join(directory, name)); +} + +function extractTaskId(file) { + const match = path.basename(file).match(/-(\d+)\.xlsx$/); + assert(match, `无法从文件名解析任务号：${file}`); + return Number(match[1]); +} + +function buildMonths() { + const result = []; + for (let year = 2022, month = 1; year < 2026 || (year === 2026 && month <= 7); ) { + const naturalLastDay = new Date(Date.UTC(year, month, 0)).getUTCDate(); + const lastDay = year === 2026 && month === 7 ? 20 : naturalLastDay; + result.push({ + key: `${year}-${pad(month)}`, + year, + month, + dayCount: lastDay, + dateFrom: `${year}-${pad(month)}-01`, + dateTo: `${year}-${pad(month)}-${pad(lastDay)}`, + }); + month += 1; + if (month === 13) { + year += 1; + month = 1; + } + } + return result; +} + +async function readPriorMapping(csvPath) { + const text = await fs.readFile(csvPath, "utf8"); + const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/); + const result = new Map(); + for (const line of lines.slice(1)) { + if (!line) continue; + const columns = parseCsvLine(line); + const [vin, plate, phone] = columns; + if (!vin || !plate) continue; + const current = result.get(plate); + if (current && current.vin !== vin) { + result.delete(plate); + continue; + } + result.set(plate, { vin, phone: phone ?? "" }); + } + return result; +} + +async function writeImportCsv(file, vehicleRows, dates) { + const stream = fsSync.createWriteStream(file, { encoding: "utf8" }); + stream.write("\uFEFFplate,date,daily_mileage_km\n"); + for (const vehicle of vehicleRows) { + for (const date of dates) { + const value = vehicle.values.get(date) ?? 0; + if (!stream.write(`${csvCell(vehicle.plate)},${date},${numberText(value)}\n`)) { + await once(stream, "drain"); + } + } + } + stream.end(); + await once(stream, "finish"); +} + +function headerFormat() { + return { + fill: "#0F766E", + font: { bold: true, color: "#FFFFFF" }, + horizontalAlignment: "center", + verticalAlignment: "center", + wrapText: true, + borders: { preset: "outside", style: "thin", color: "#115E59" }, + }; +} + +function findHeader(headers, names) { + return headers.findIndex((header) => names.includes(header)); +} + +function parseCsvLine(line) { + const result = []; + let value = ""; + let quoted = false; + for (let index = 0; index < line.length; index += 1) { + const char = line[index]; + if (quoted) { + if (char === '"' && line[index + 1] === '"') { + value += '"'; + index += 1; + } else if (char === '"') { + quoted = false; + } else { + value += char; + } + } else if (char === '"') { + quoted = true; + } else if (char === ",") { + result.push(value); + value = ""; + } else { + value += char; + } + } + result.push(value); + return result; +} + +function csvCell(value) { + const text = cleanText(value); + return /[",\r\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; +} + +function cleanText(value) { + return value === null || value === undefined ? "" : String(value).trim(); +} + +function numberValue(value) { + const number = Number(value); + return Number.isFinite(number) ? number : 0; +} + +function numberText(value) { + return String(round(value)); +} + +function round(value) { + return Math.round((value + Number.EPSILON) * 100) / 100; +} + +function pad(value) { + return String(value).padStart(2, "0"); +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} diff --git a/tmp/g7-mileage-history-20260724/importer.go b/tmp/g7-mileage-history-20260724/importer.go new file mode 100644 index 00000000..51f71188 --- /dev/null +++ b/tmp/g7-mileage-history-20260724/importer.go @@ -0,0 +1,787 @@ +package main + +import ( + "compress/gzip" + "context" + "database/sql" + "encoding/csv" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + mysql "github.com/go-sql-driver/mysql" +) + +const ( + g7Protocol = "JT808" + g7SourceID = int64(661200) + g7SourceIP = "manual-g7s-excel" + g7SourceCode = "g7s_excel" + g7PlatformName = "G7s Excel Override" + g7QualityReason = "manual_g7s_gps_quarterly_import_20260101_20260630_20260728" + dateFrom = "2026-01-01" + dateTo = "2026-06-30" + expectedDays = 181 +) + +type mapping struct { + VIN string + Ambiguous bool +} + +type sourceStats struct { + Rows int64 `json:"rows"` + Vehicles int `json:"vehicles"` + Dates int `json:"dates"` + Positive int64 `json:"positive_rows"` + Zero int64 `json:"zero_rows"` + TotalKM float64 `json:"total_km"` + MinDate string `json:"min_date"` + MaxDate string `json:"max_date"` + PlateCount int `json:"plate_count"` +} + +type stageStats struct { + Rows int64 `json:"rows"` + Vehicles int `json:"vehicles"` + Dates int `json:"dates"` + Positive int64 `json:"positive_rows"` + Zero int64 `json:"zero_rows"` + TotalKM float64 `json:"total_km"` + MappedPlates int `json:"mapped_plates"` + UnmappedPlates int `json:"unmapped_plates"` + AmbiguousPlates int `json:"ambiguous_plates"` + VINCollisionPlates int `json:"vin_collision_plates"` +} + +type existingStats struct { + ExistingFinalRows int64 `json:"existing_final_rows"` + NewFinalRows int64 `json:"new_final_rows"` + ChangedExistingFinalRows int64 `json:"changed_existing_final_rows"` + UnchangedExistingFinalRows int64 `json:"unchanged_existing_final_rows"` + OtherSourceFinalRows int64 `json:"other_source_final_rows"` + OtherSelectedCandidateRows int64 `json:"other_selected_candidate_rows"` + ExistingFinalTotalKM float64 `json:"existing_final_total_km"` +} + +type result struct { + Mode string `json:"mode"` + StartedAt string `json:"started_at"` + CompletedAt string `json:"completed_at"` + Source sourceStats `json:"source"` + Stage stageStats `json:"stage"` + Existing existingStats `json:"existing"` + UnmappedPlates []string `json:"unmapped_plates"` + AmbiguousPlates []string `json:"ambiguous_plates"` + CollisionPlates []string `json:"vin_collision_plates"` + BackupFinal string `json:"backup_final,omitempty"` + BackupCandidates string `json:"backup_candidates,omitempty"` + FinalMatchedRows int64 `json:"final_matched_rows"` + FinalTotalKM float64 `json:"final_total_km"` + CandidateMatchedRows int64 `json:"candidate_matched_rows"` + OtherSelectedRows int64 `json:"other_selected_rows"` + SourceID int64 `json:"source_id"` +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + var inputPath, mappingPath, dsn, backupDir string + var apply bool + var timeout time.Duration + flag.StringVar(&inputPath, "input", "", "normalized CSV or CSV.gz") + flag.StringVar(&mappingPath, "mapping-csv", "", "optional existing vin,plate mapping CSV") + flag.StringVar(&dsn, "mysql-dsn", strings.TrimSpace(os.Getenv("MYSQL_DSN")), "MySQL DSN") + flag.StringVar(&backupDir, "backup-dir", "/tmp/g7-gps-quarterly-backup-20260728", "backup directory for apply") + flag.BoolVar(&apply, "apply", false, "apply changes; default is dry-run") + flag.DurationVar(&timeout, "timeout", 30*time.Minute, "overall timeout") + flag.Parse() + + if strings.TrimSpace(inputPath) == "" { + return errors.New("-input is required") + } + if strings.TrimSpace(dsn) == "" { + return errors.New("MYSQL_DSN or -mysql-dsn is required") + } + normalized, err := normalizedDSN(dsn) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + db, err := sql.Open("mysql", normalized) + if err != nil { + return err + } + defer db.Close() + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(2) + if err := db.PingContext(ctx); err != nil { + return err + } + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + + started := time.Now() + source, inputPlates, err := scanInput(inputPath) + if err != nil { + return err + } + if source.MinDate != dateFrom || source.MaxDate != dateTo || source.Dates != expectedDays { + return fmt.Errorf("unexpected input range: %s..%s (%d dates)", source.MinDate, source.MaxDate, source.Dates) + } + mappings, err := loadMappings(ctx, conn) + if err != nil { + return err + } + if mappingPath != "" { + if err := mergeMappingCSV(mappingPath, mappings); err != nil { + return err + } + } + resolution := resolveInputPlates(inputPlates, mappings) + if err := createStage(ctx, conn); err != nil { + return err + } + stage, err := loadStage(ctx, conn, inputPath, resolution) + if err != nil { + return err + } + stage.MappedPlates = len(resolution.Mapped) + stage.UnmappedPlates = len(resolution.Unmapped) + stage.AmbiguousPlates = len(resolution.Ambiguous) + stage.VINCollisionPlates = len(resolution.Collisions) + if stage.Rows != int64(stage.MappedPlates*expectedDays) { + return fmt.Errorf("stage row count mismatch: got %d, expected %d", stage.Rows, stage.MappedPlates*expectedDays) + } + existing, err := inspectExisting(ctx, conn) + if err != nil { + return err + } + out := result{ + Mode: "dry_run", + StartedAt: started.Format(time.RFC3339), + Source: source, + Stage: stage, + Existing: existing, + UnmappedPlates: resolution.Unmapped, + AmbiguousPlates: resolution.Ambiguous, + CollisionPlates: resolution.Collisions, + SourceID: g7SourceID, + } + + if apply { + out.Mode = "apply" + if err := os.MkdirAll(backupDir, 0o700); err != nil { + return err + } + out.BackupFinal = filepath.Join(backupDir, "vehicle_daily_mileage_before.csv.gz") + out.BackupCandidates = filepath.Join(backupDir, "vehicle_daily_mileage_source_before.csv.gz") + if err := backupQuery(ctx, conn, out.BackupFinal, ` +SELECT m.* +FROM vehicle_daily_mileage m +JOIN tmp_g7_mileage_import s + ON s.vin = m.vin AND s.stat_date = m.stat_date +WHERE m.protocol = 'JT808' +ORDER BY m.vin, m.stat_date`); err != nil { + return fmt.Errorf("backup final rows: %w", err) + } + if err := backupQuery(ctx, conn, out.BackupCandidates, ` +SELECT c.* +FROM vehicle_daily_mileage_source c +JOIN tmp_g7_mileage_import s + ON s.vin = c.vin AND s.stat_date = c.stat_date +WHERE c.protocol = 'JT808' +ORDER BY c.vin, c.stat_date, c.source_key`); err != nil { + return fmt.Errorf("backup candidate rows: %w", err) + } + finalRows, finalTotal, candidateRows, otherSelected, err := applyImport(ctx, conn) + if err != nil { + return err + } + out.FinalMatchedRows = finalRows + out.FinalTotalKM = finalTotal + out.CandidateMatchedRows = candidateRows + out.OtherSelectedRows = otherSelected + } + out.CompletedAt = time.Now().Format(time.RFC3339) + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + return encoder.Encode(out) +} + +type plateResolution struct { + Mapped map[string]string + Unmapped []string + Ambiguous []string + Collisions []string +} + +func resolveInputPlates(plates map[string]struct{}, mappings map[string]mapping) plateResolution { + result := plateResolution{Mapped: make(map[string]string)} + vinToPlates := make(map[string][]string) + for plate := range plates { + value, ok := mappings[plate] + switch { + case !ok: + result.Unmapped = append(result.Unmapped, plate) + case value.Ambiguous: + result.Ambiguous = append(result.Ambiguous, plate) + default: + result.Mapped[plate] = value.VIN + vinToPlates[value.VIN] = append(vinToPlates[value.VIN], plate) + } + } + for _, presentPlates := range vinToPlates { + if len(presentPlates) < 2 { + continue + } + for _, plate := range presentPlates { + delete(result.Mapped, plate) + result.Collisions = append(result.Collisions, plate) + } + } + sort.Strings(result.Unmapped) + sort.Strings(result.Ambiguous) + sort.Strings(result.Collisions) + return result +} + +func scanInput(inputPath string) (sourceStats, map[string]struct{}, error) { + reader, closeReader, err := openCSV(inputPath) + if err != nil { + return sourceStats{}, nil, err + } + defer closeReader() + csvReader := csv.NewReader(reader) + header, err := csvReader.Read() + if err != nil { + return sourceStats{}, nil, err + } + if len(header) < 3 { + return sourceStats{}, nil, errors.New("input CSV needs plate,date,daily_mileage_km") + } + header[0] = strings.TrimPrefix(header[0], "\uFEFF") + if header[0] != "plate" || header[1] != "date" || header[2] != "daily_mileage_km" { + return sourceStats{}, nil, fmt.Errorf("unexpected CSV header: %v", header) + } + plates := make(map[string]struct{}) + dates := make(map[string]struct{}) + stats := sourceStats{} + for { + record, readErr := csvReader.Read() + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return sourceStats{}, nil, readErr + } + if len(record) < 3 { + return sourceStats{}, nil, fmt.Errorf("short CSV row at %d", stats.Rows+2) + } + plate := strings.TrimSpace(record[0]) + date := strings.TrimSpace(record[1]) + mileage, parseErr := strconv.ParseFloat(strings.TrimSpace(record[2]), 64) + if parseErr != nil || mileage < 0 || mileage > 2500 { + return sourceStats{}, nil, fmt.Errorf("invalid mileage at row %d: %q", stats.Rows+2, record[2]) + } + if _, parseErr = time.Parse("2006-01-02", date); parseErr != nil { + return sourceStats{}, nil, fmt.Errorf("invalid date at row %d: %q", stats.Rows+2, date) + } + if plate == "" { + return sourceStats{}, nil, fmt.Errorf("empty plate at row %d", stats.Rows+2) + } + plates[plate] = struct{}{} + dates[date] = struct{}{} + stats.Rows++ + stats.TotalKM += mileage + if mileage > 0 { + stats.Positive++ + } else { + stats.Zero++ + } + if stats.MinDate == "" || date < stats.MinDate { + stats.MinDate = date + } + if date > stats.MaxDate { + stats.MaxDate = date + } + } + stats.PlateCount = len(plates) + stats.Vehicles = len(plates) + stats.Dates = len(dates) + stats.TotalKM = round(stats.TotalKM) + return stats, plates, nil +} + +func loadMappings(ctx context.Context, conn *sql.Conn) (map[string]mapping, error) { + rows, err := conn.QueryContext(ctx, ` +SELECT plate, vin +FROM ( + SELECT TRIM(plate) AS plate, TRIM(vin) AS vin + FROM vehicle + WHERE plate IS NOT NULL AND TRIM(plate) <> '' + AND vin IS NOT NULL AND TRIM(vin) <> '' + UNION ALL + SELECT TRIM(plate) AS plate, TRIM(vin) AS vin + FROM vehicle_identifier + WHERE enabled = 1 AND plate IS NOT NULL AND TRIM(plate) <> '' + AND vin IS NOT NULL AND TRIM(vin) <> '' +) mappings`) + if err != nil { + return nil, err + } + defer rows.Close() + result := make(map[string]mapping) + for rows.Next() { + var plate, vin string + if err := rows.Scan(&plate, &vin); err != nil { + return nil, err + } + current, ok := result[plate] + if !ok { + result[plate] = mapping{VIN: vin} + } else if current.VIN != vin { + current.Ambiguous = true + result[plate] = current + } + } + return result, rows.Err() +} + +func mergeMappingCSV(inputPath string, mappings map[string]mapping) error { + file, err := os.Open(inputPath) + if err != nil { + return err + } + defer file.Close() + reader := csv.NewReader(file) + header, err := reader.Read() + if err != nil { + return err + } + if len(header) < 2 { + return errors.New("mapping CSV must contain vin,plate") + } + header[0] = strings.TrimPrefix(header[0], "\uFEFF") + if header[0] != "vin" || header[1] != "plate" { + return fmt.Errorf("unexpected mapping CSV header: %v", header[:2]) + } + for { + record, readErr := reader.Read() + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return readErr + } + if len(record) < 2 { + continue + } + vin := strings.TrimSpace(record[0]) + plate := strings.TrimSpace(record[1]) + if vin == "" || plate == "" { + continue + } + current, ok := mappings[plate] + if !ok { + mappings[plate] = mapping{VIN: vin} + } else if current.VIN != vin { + current.Ambiguous = true + mappings[plate] = current + } + } + return nil +} + +func createStage(ctx context.Context, conn *sql.Conn) error { + _, err := conn.ExecContext(ctx, ` +DROP TEMPORARY TABLE IF EXISTS tmp_g7_mileage_import`) + if err != nil { + return err + } + _, err = conn.ExecContext(ctx, ` +CREATE TEMPORARY TABLE tmp_g7_mileage_import ( + vin varchar(32) NOT NULL, + plate varchar(32) NOT NULL, + stat_date date NOT NULL, + daily_mileage_km decimal(18,3) NOT NULL, + PRIMARY KEY (vin, stat_date), + KEY idx_tmp_plate (plate), + KEY idx_tmp_date (stat_date) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`) + return err +} + +func loadStage(ctx context.Context, conn *sql.Conn, inputPath string, resolution plateResolution) (stageStats, error) { + reader, closeReader, err := openCSV(inputPath) + if err != nil { + return stageStats{}, err + } + defer closeReader() + csvReader := csv.NewReader(reader) + if _, err := csvReader.Read(); err != nil { + return stageStats{}, err + } + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return stageStats{}, err + } + defer tx.Rollback() + + type stageRow struct { + VIN, Plate, Date string + Mileage float64 + } + batch := make([]stageRow, 0, 1000) + flush := func() error { + if len(batch) == 0 { + return nil + } + var query strings.Builder + query.WriteString("INSERT INTO tmp_g7_mileage_import (vin, plate, stat_date, daily_mileage_km) VALUES ") + args := make([]any, 0, len(batch)*4) + for index, row := range batch { + if index > 0 { + query.WriteByte(',') + } + query.WriteString("(?,?,?,?)") + args = append(args, row.VIN, row.Plate, row.Date, row.Mileage) + } + if _, execErr := tx.ExecContext(ctx, query.String(), args...); execErr != nil { + return execErr + } + batch = batch[:0] + return nil + } + for { + record, readErr := csvReader.Read() + if errors.Is(readErr, io.EOF) { + break + } + if readErr != nil { + return stageStats{}, readErr + } + plate := strings.TrimSpace(record[0]) + vin, ok := resolution.Mapped[plate] + if !ok { + continue + } + mileage, _ := strconv.ParseFloat(strings.TrimSpace(record[2]), 64) + batch = append(batch, stageRow{ + VIN: vin, Plate: plate, Date: strings.TrimSpace(record[1]), Mileage: mileage, + }) + if len(batch) == cap(batch) { + if err := flush(); err != nil { + return stageStats{}, err + } + } + } + if err := flush(); err != nil { + return stageStats{}, err + } + if err := tx.Commit(); err != nil { + return stageStats{}, err + } + var stats stageStats + var nullableSum sql.NullFloat64 + err = conn.QueryRowContext(ctx, ` +SELECT COUNT(*), COUNT(DISTINCT vin), COUNT(DISTINCT stat_date), + SUM(daily_mileage_km > 0), SUM(daily_mileage_km = 0), + SUM(daily_mileage_km) +FROM tmp_g7_mileage_import`).Scan( + &stats.Rows, &stats.Vehicles, &stats.Dates, &stats.Positive, &stats.Zero, &nullableSum, + ) + if nullableSum.Valid { + stats.TotalKM = round(nullableSum.Float64) + } + return stats, err +} + +func inspectExisting(ctx context.Context, conn *sql.Conn) (existingStats, error) { + var stats existingStats + var nullableSum sql.NullFloat64 + err := conn.QueryRowContext(ctx, ` +SELECT + SUM(m.vin IS NOT NULL), + SUM(m.vin IS NULL), + SUM(m.vin IS NOT NULL AND ABS(m.daily_mileage_km - s.daily_mileage_km) > 0.0005), + SUM(m.vin IS NOT NULL AND ABS(m.daily_mileage_km - s.daily_mileage_km) <= 0.0005), + SUM(m.vin IS NOT NULL AND COALESCE(m.source_id, 0) <> ?), + SUM(CASE WHEN m.vin IS NOT NULL THEN m.daily_mileage_km ELSE 0 END) +FROM tmp_g7_mileage_import s +LEFT JOIN vehicle_daily_mileage m + ON m.vin = s.vin AND m.stat_date = s.stat_date AND m.protocol = 'JT808'`, g7SourceID).Scan( + &stats.ExistingFinalRows, + &stats.NewFinalRows, + &stats.ChangedExistingFinalRows, + &stats.UnchangedExistingFinalRows, + &stats.OtherSourceFinalRows, + &nullableSum, + ) + if err != nil { + return stats, err + } + if nullableSum.Valid { + stats.ExistingFinalTotalKM = round(nullableSum.Float64) + } + err = conn.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM vehicle_daily_mileage_source c +JOIN tmp_g7_mileage_import s + ON s.vin = c.vin AND s.stat_date = c.stat_date +WHERE c.protocol = 'JT808' AND c.is_selected = 1 + AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel')`).Scan( + &stats.OtherSelectedCandidateRows, + ) + return stats, err +} + +func applyImport(ctx context.Context, conn *sql.Conn) (int64, float64, int64, int64, error) { + tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted}) + if err != nil { + return 0, 0, 0, 0, err + } + defer tx.Rollback() + if _, err := tx.ExecContext(ctx, ` +UPDATE vehicle_daily_mileage_source c +JOIN tmp_g7_mileage_import s + ON s.vin = c.vin AND s.stat_date = c.stat_date +SET c.is_selected = 0 +WHERE c.protocol = 'JT808' + AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel') + AND c.is_selected <> 0`); err != nil { + return 0, 0, 0, 0, fmt.Errorf("clear other selected candidates: %w", err) + } + if _, err := tx.ExecContext(ctx, ` +INSERT INTO vehicle_daily_mileage_source ( + vin, stat_date, protocol, source_key, source_ip, source_endpoint, + platform_name, first_total_mileage_km, latest_total_mileage_km, + daily_mileage_km, sample_count, first_event_time, latest_event_time, + quality_status, quality_reason, is_selected +) +SELECT + s.vin, s.stat_date, 'JT808', + CONCAT('JT808:', s.vin, '@PLATFORM:g7s_excel'), + 'manual-g7s-excel', 'manual-g7s-excel', + 'G7s Excel Override', NULL, NULL, + s.daily_mileage_km, 1, + TIMESTAMP(s.stat_date, '00:00:00'), + TIMESTAMP(s.stat_date, '23:59:59'), + 'OK', ?, 1 +FROM tmp_g7_mileage_import s +ON DUPLICATE KEY UPDATE + source_ip = VALUES(source_ip), + source_endpoint = VALUES(source_endpoint), + platform_name = VALUES(platform_name), + first_total_mileage_km = NULL, + latest_total_mileage_km = NULL, + daily_mileage_km = VALUES(daily_mileage_km), + sample_count = VALUES(sample_count), + first_event_time = VALUES(first_event_time), + latest_event_time = VALUES(latest_event_time), + quality_status = VALUES(quality_status), + quality_reason = VALUES(quality_reason), + is_selected = 1`, g7QualityReason); err != nil { + return 0, 0, 0, 0, fmt.Errorf("upsert G7 candidates: %w", err) + } + if _, err := tx.ExecContext(ctx, ` +INSERT INTO vehicle_daily_mileage ( + vin, stat_date, protocol, source_id, daily_mileage_km, + latest_total_mileage_km, updated_at +) +SELECT vin, stat_date, 'JT808', ?, daily_mileage_km, NULL, NOW() +FROM tmp_g7_mileage_import +ON DUPLICATE KEY UPDATE + source_id = VALUES(source_id), + daily_mileage_km = VALUES(daily_mileage_km), + latest_total_mileage_km = NULL, + updated_at = NOW()`, g7SourceID); err != nil { + return 0, 0, 0, 0, fmt.Errorf("upsert final mileage: %w", err) + } + if _, err := tx.ExecContext(ctx, ` +UPDATE vehicle_data_source +SET latest_seen_at = GREATEST(COALESCE(latest_seen_at, '2026-06-30 23:59:59'), '2026-06-30 23:59:59'), + first_seen_at = LEAST(COALESCE(first_seen_at, '2026-01-01 00:00:00'), '2026-01-01 00:00:00'), + latest_source_endpoint = 'manual-g7s-excel', + platform_name = 'G7s Excel Override', + source_code = 'g7s_excel', + source_kind = 'PLATFORM', + trust_priority = 0, + enabled = 1, + remark = 'Manual GPS override imported from G7s quarterly daily mileage reports for 2026-01-01 through 2026-06-30 on 2026-07-28' +WHERE id = ? AND protocol = 'JT808' AND source_ip = 'manual-g7s-excel'`, g7SourceID); err != nil { + return 0, 0, 0, 0, fmt.Errorf("update G7 source metadata: %w", err) + } + var finalRows, candidateRows, otherSelected int64 + var finalSum sql.NullFloat64 + if err := tx.QueryRowContext(ctx, ` +SELECT COUNT(*), SUM(m.daily_mileage_km) +FROM vehicle_daily_mileage m +JOIN tmp_g7_mileage_import s + ON s.vin = m.vin AND s.stat_date = m.stat_date +WHERE m.protocol = 'JT808' AND m.source_id = ? + AND ABS(m.daily_mileage_km - s.daily_mileage_km) <= 0.0005`, g7SourceID).Scan( + &finalRows, &finalSum, + ); err != nil { + return 0, 0, 0, 0, err + } + if err := tx.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM vehicle_daily_mileage_source c +JOIN tmp_g7_mileage_import s + ON s.vin = c.vin AND s.stat_date = c.stat_date +WHERE c.protocol = 'JT808' + AND c.source_key = CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel') + AND c.is_selected = 1 + AND ABS(c.daily_mileage_km - s.daily_mileage_km) <= 0.0005`).Scan(&candidateRows); err != nil { + return 0, 0, 0, 0, err + } + if err := tx.QueryRowContext(ctx, ` +SELECT COUNT(*) +FROM vehicle_daily_mileage_source c +JOIN tmp_g7_mileage_import s + ON s.vin = c.vin AND s.stat_date = c.stat_date +WHERE c.protocol = 'JT808' + AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel') + AND c.is_selected = 1`).Scan(&otherSelected); err != nil { + return 0, 0, 0, 0, err + } + var stageRows int64 + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM tmp_g7_mileage_import`).Scan(&stageRows); err != nil { + return 0, 0, 0, 0, err + } + if finalRows != stageRows || candidateRows != stageRows || otherSelected != 0 { + return 0, 0, 0, 0, fmt.Errorf( + "verification failed before commit: stage=%d final=%d candidates=%d other_selected=%d", + stageRows, finalRows, candidateRows, otherSelected, + ) + } + if err := tx.Commit(); err != nil { + return 0, 0, 0, 0, err + } + return finalRows, round(finalSum.Float64), candidateRows, otherSelected, nil +} + +func backupQuery(ctx context.Context, conn *sql.Conn, outputPath, query string) error { + rows, err := conn.QueryContext(ctx, query) + if err != nil { + return err + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + return err + } + file, err := os.OpenFile(outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer file.Close() + gzipWriter := gzip.NewWriter(file) + defer gzipWriter.Close() + csvWriter := csv.NewWriter(gzipWriter) + defer csvWriter.Flush() + if err := csvWriter.Write(columns); err != nil { + return err + } + raw := make([]sql.RawBytes, len(columns)) + pointers := make([]any, len(columns)) + for index := range raw { + pointers[index] = &raw[index] + } + for rows.Next() { + if err := rows.Scan(pointers...); err != nil { + return err + } + record := make([]string, len(columns)) + for index, value := range raw { + if value == nil { + record[index] = "\\N" + } else { + record[index] = string(value) + } + } + if err := csvWriter.Write(record); err != nil { + return err + } + } + if err := rows.Err(); err != nil { + return err + } + csvWriter.Flush() + if err := csvWriter.Error(); err != nil { + return err + } + if err := gzipWriter.Close(); err != nil { + return err + } + return file.Sync() +} + +func openCSV(inputPath string) (io.Reader, func(), error) { + file, err := os.Open(inputPath) + if err != nil { + return nil, nil, err + } + closeReader := func() { _ = file.Close() } + if !strings.HasSuffix(strings.ToLower(inputPath), ".gz") { + return file, closeReader, nil + } + gzipReader, err := gzip.NewReader(file) + if err != nil { + file.Close() + return nil, nil, err + } + return gzipReader, func() { + _ = gzipReader.Close() + _ = file.Close() + }, nil +} + +func normalizedDSN(raw string) (string, error) { + config, err := mysql.ParseDSN(raw) + if err != nil { + return "", err + } + location, err := time.LoadLocation("Asia/Shanghai") + if err != nil { + return "", err + } + config.ParseTime = true + config.Loc = location + config.Timeout = 10 * time.Second + config.ReadTimeout = 30 * time.Minute + config.WriteTimeout = 30 * time.Minute + config.Params = cloneParams(config.Params) + config.Params["charset"] = "utf8mb4" + return config.FormatDSN(), nil +} + +func cloneParams(input map[string]string) map[string]string { + output := make(map[string]string, len(input)+1) + for key, value := range input { + output[key] = value + } + return output +} + +func round(value float64) float64 { + return float64(int64(value*100+0.5)) / 100 +} diff --git a/tmp/g7-mileage-history-20260724/schema_probe.go b/tmp/g7-mileage-history-20260724/schema_probe.go new file mode 100644 index 00000000..3dce56c7 --- /dev/null +++ b/tmp/g7-mileage-history-20260724/schema_probe.go @@ -0,0 +1,66 @@ +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "os" + "strings" + + _ "github.com/go-sql-driver/mysql" +) + +func main() { + dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")) + if dsn == "" { + panic("MYSQL_DSN is required") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + panic(err) + } + defer db.Close() + for _, table := range []string{ + "vehicle_data_source", + "vehicle_daily_mileage_source", + "vehicle_daily_mileage", + "vehicle", + "vehicle_identifier", + } { + var name, ddl string + if err := db.QueryRow("SHOW CREATE TABLE "+table).Scan(&name, &ddl); err != nil { + panic(err) + } + fmt.Printf("=== %s ===\n%s\n", name, ddl) + } + rows, err := db.Query("SELECT * FROM vehicle_data_source WHERE id = 661200") + if err != nil { + panic(err) + } + defer rows.Close() + columns, err := rows.Columns() + if err != nil { + panic(err) + } + if rows.Next() { + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + panic(err) + } + record := make(map[string]any, len(columns)) + for i, column := range columns { + switch value := values[i].(type) { + case []byte: + record[column] = string(value) + default: + record[column] = value + } + } + data, _ := json.Marshal(record) + fmt.Printf("=== source_661200 ===\n%s\n", data) + } +} diff --git a/tmp/g7-mileage-history-20260724/verify_final.mjs b/tmp/g7-mileage-history-20260724/verify_final.mjs new file mode 100644 index 00000000..b26c5d78 --- /dev/null +++ b/tmp/g7-mileage-history-20260724/verify_final.mjs @@ -0,0 +1,41 @@ +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const inputPath = + "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/g7-mileage-history-20260724/merged/G7车辆每日里程及里程统计_20220101-20260720.xlsx"; +const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(inputPath)); +const sheets = await workbook.inspect({ + kind: "sheet", + include: "id,name", + maxChars: 4000, +}); +const summary = await workbook.inspect({ + kind: "table", + sheetId: "汇总", + range: "A1:H25", + include: "values,formulas", + maxChars: 12000, + tableMaxRows: 25, + tableMaxCols: 8, +}); +const daily = await workbook.inspect({ + kind: "table", + sheetId: "每日里程非零", + range: "A1:D12", + include: "values,formulas", + maxChars: 6000, + tableMaxRows: 12, + tableMaxCols: 4, +}); +const errors = await workbook.inspect({ + kind: "match", + searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", + options: { useRegex: true, maxResults: 300 }, + summary: "post-export formula error scan", +}); +console.log(JSON.stringify({ + inputPath, + sheets: sheets.ndjson, + summary: summary.ndjson, + daily: daily.ndjson, + errors: errors.ndjson, +}, null, 2)); diff --git a/tmp/gb32960-export-20260817/build_export.mjs b/tmp/gb32960-export-20260817/build_export.mjs new file mode 100644 index 00000000..c3a9da1e --- /dev/null +++ b/tmp/gb32960-export-20260817/build_export.mjs @@ -0,0 +1,364 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { SpreadsheetFile, Workbook } from "@oai/artifact-tool"; + +const taskDir = process.env.GB32960_TASK_DIR || "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/gb32960-export-20260817"; +const outputDir = process.env.GB32960_OUTPUT_DIR || "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/gb32960-export-20260817-agp4377"; +const data = JSON.parse(await fs.readFile(path.join(taskDir, "raw-frames.json"), "utf8")); +const exportDate = data.query.dateFrom.slice(0, 10); +const outputPath = path.join(outputDir, `${data.query.plate}_${exportDate}_GB32960原始数据_中文字段.xlsx`); + +const categoryNames = { + alarm: "报警数据", + drive_motor: "驱动电机数据", + engine: "发动机数据", + extreme: "极值数据", + fuel_cell: "燃料电池数据", + gd_fc_air_conditioner: "广东燃料电池空调数据", + gd_fc_auxiliary: "广东燃料电池辅助系统数据", + gd_fc_dcdc: "广东燃料电池 DC/DC 数据", + gd_fc_demo_extension: "广东燃料电池演示扩展数据", + gd_fc_stack: "广东燃料电池电堆数据", + gd_fc_vehicle_info: "广东燃料电池车辆信息数据", + gd_fc_vendor_tlv: "广东燃料电池厂家 TLV 扩展", + header: "报文头", + platform: "平台信息", + position: "车辆位置数据", + temperature: "可充电储能装置温度数据", + vehicle: "整车数据", + voltage: "可充电储能装置电压数据", +}; + +const fieldNames = { + battery_faults: "可充电储能装置故障码", + engine_faults: "发动机故障码", + general_alarm_flag: "通用报警标志", + max_alarm_level: "最高报警等级", + motor_faults: "驱动电机故障码", + other_faults: "其他故障码", + type: "数据单元类型", + count: "数量", + controller_current_a: "控制器直流母线电流", + controller_temperature_c: "控制器温度", + controller_voltage_v: "控制器输入电压", + motor_temperature_c: "电机温度", + serial_no: "驱动电机序号", + speed_rpm: "转速", + state: "状态", + torque_nm: "转矩", + engine_status: "发动机状态", + fuel_rate: "燃料消耗率", + max_temp_c: "最高温度", + max_temp_probe_no: "最高温度探针序号", + max_temp_subsystem_no: "最高温度子系统号", + max_voltage_cell_no: "最高电压单体序号", + max_voltage_subsystem_no: "最高电压子系统号", + max_voltage_v: "最高电压", + min_temp_c: "最低温度", + min_temp_probe_no: "最低温度探针序号", + min_temp_subsystem_no: "最低温度子系统号", + min_voltage_cell_no: "最低电压单体序号", + min_voltage_subsystem_no: "最低电压子系统号", + min_voltage_v: "最低电压", + dc_dc_status: "高压 DC/DC 状态", + fuel_cell_current_a: "燃料电池电流", + fuel_cell_voltage_v: "燃料电池电压", + hydrogen_consumption_kg_per_100km: "氢耗", + max_hydrogen_concentration_fraction: "最高氢浓度比例", + max_hydrogen_concentration_percent: "最高氢浓度", + max_hydrogen_concentration_ppm: "最高氢浓度原始值", + max_hydrogen_concentration_probe_id: "最高氢浓度探针编号", + max_hydrogen_pressure_mpa: "最高氢压力", + max_hydrogen_pressure_probe_id: "最高氢压探针编号", + max_hydrogen_temperature_c: "氢系统最高温度", + max_hydrogen_temperature_probe_id: "最高氢温探针编号", + temperature_probe_count: "温度探针数量", + temperature_probe_values_c: "温度探针值列表", + status: "状态", + subsystem_count: "子系统数量", + air_compressor_motor_voltage_v: "空气压缩机电机电压", + air_compressor_power_kw: "空气压缩机功率", + low_voltage_battery_voltage_v: "低压蓄电池电压", + water_pump_voltage_v: "水泵电压", + controller_temp_c: "控制器温度", + input_current_a: "输入电流", + input_voltage_v: "输入电压", + output_current_a: "输出电流", + output_voltage_v: "输出电压", + air_compressor_current_a: "空气压缩机电流", + air_compressor_voltage_v: "空气压缩机电压", + declared_length: "声明长度", + hydrogen_pump_current_a: "氢气循环泵电流", + hydrogen_pump_voltage_v: "氢气循环泵电压", + stack_temp_c: "电堆温度", + air_inlet_pressure_kpa: "空气入口压力", + avg_cell_voltage_v: "单体平均电压", + cell_count: "单体总数", + engine_work_state: "发动机工作状态", + frame_cell_voltages_v: "本帧单体电压列表", + hydrogen_inlet_pressure_kpa: "氢气入口压力", + max_cell_voltage_id: "最高电压单体编号", + max_cell_voltage_v: "单体最高电压", + min_cell_voltage_id: "最低电压单体编号", + min_cell_voltage_v: "单体最低电压", + stack_count: "电堆数量", + stack_water_outlet_temp_c: "电堆出水温度", + collision_alarm: "碰撞报警", + hydrogen_mass_kg: "车载氢量", + payload_hex: "原始载荷（十六进制）", + actual_body_length: "实际报文体长度", + body_length: "报文体长度", + command: "命令标识", + encrypt: "加密方式", + response_flag: "应答标志", + version: "协议版本", + vin: "车辆识别代号", + platform_name: "平台名称", + latitude: "纬度", + longitude: "经度", + position_status: "定位状态标志", + accelerator_pct: "加速踏板行程", + brake_pct: "制动踏板状态", + charge_status: "充电状态", + gear: "挡位原始值", + insulation_kohm: "绝缘电阻", + running_mode: "运行模式", + soc_percent: "SOC", + speed_kmh: "车速", + total_current_a: "总电流", + total_mileage_km: "累计里程", + total_voltage_v: "总电压", + vehicle_status: "车辆状态", + max_cell_v: "单体最高电压", + min_cell_v: "单体最低电压", +}; + +const unitBySuffix = { + controller_current_a: "A", controller_voltage_v: "V", controller_temperature_c: "℃", + motor_temperature_c: "℃", speed_rpm: "rpm", torque_nm: "N·m", fuel_rate: "L/100km", + max_temp_c: "℃", min_temp_c: "℃", max_voltage_v: "V", min_voltage_v: "V", + fuel_cell_current_a: "A", fuel_cell_voltage_v: "V", hydrogen_consumption_kg_per_100km: "kg/100km", + max_hydrogen_concentration_fraction: "比例", max_hydrogen_concentration_percent: "%", max_hydrogen_concentration_ppm: "ppm", + max_hydrogen_pressure_mpa: "MPa", max_hydrogen_temperature_c: "℃", temperature_probe_values_c: "℃", + air_compressor_motor_voltage_v: "V", air_compressor_power_kw: "kW", low_voltage_battery_voltage_v: "V", water_pump_voltage_v: "V", + controller_temp_c: "℃", input_current_a: "A", input_voltage_v: "V", output_current_a: "A", output_voltage_v: "V", + air_compressor_current_a: "A", air_compressor_voltage_v: "V", hydrogen_pump_current_a: "A", hydrogen_pump_voltage_v: "V", stack_temp_c: "℃", + air_inlet_pressure_kpa: "kPa", avg_cell_voltage_v: "V", frame_cell_voltages_v: "V", hydrogen_inlet_pressure_kpa: "kPa", + max_cell_voltage_v: "V", min_cell_voltage_v: "V", stack_water_outlet_temp_c: "℃", hydrogen_mass_kg: "kg", + latitude: "°", longitude: "°", accelerator_pct: "%", brake_pct: "%", insulation_kohm: "kΩ", soc_percent: "%", + speed_kmh: "km/h", total_current_a: "A", total_mileage_km: "km", total_voltage_v: "V", max_cell_v: "V", min_cell_v: "V", +}; + +const valueMappings = { + "gb32960.vehicle.vehicle_status": { "1": "启动", "2": "熄火", "3": "其他", "254": "异常", "255": "无效" }, + "gb32960.vehicle.charge_status": { "1": "停车充电", "2": "行驶充电", "3": "未充电", "4": "充电完成", "254": "异常", "255": "无效" }, + "gb32960.vehicle.running_mode": { "1": "纯电", "2": "混合动力", "3": "燃料电池", "254": "异常", "255": "无效" }, + "gb32960.vehicle.dc_dc_status": { "1": "工作", "2": "断开", "254": "异常", "255": "无效" }, + "gb32960.fuel_cell.dc_dc_status": { "1": "工作", "2": "断开", "254": "异常", "255": "无效" }, + "gb32960.drive_motor.motors.motor_1.state": { "1": "耗电", "2": "发电", "3": "关闭", "4": "准备", "254": "异常", "255": "无效" }, + "gb32960.alarm.max_alarm_level": { "0": "无故障", "1": "一级故障", "2": "二级故障", "3": "三级故障" }, + "gb32960.gd_fc_air_conditioner.status": { "0": "关闭", "1": "启动", "254": "异常", "255": "无效" }, + "gb32960.gd_fc_vehicle_info.collision_alarm": { "0": "无碰撞报警", "1": "有碰撞报警", "254": "异常", "255": "无效" }, + "gb32960.position.position_status": { + "0": "有效定位 · 北纬 · 东经", "1": "无效定位 · 北纬 · 东经", "2": "有效定位 · 南纬 · 东经", "3": "无效定位 · 南纬 · 东经", + "4": "有效定位 · 北纬 · 西经", "5": "无效定位 · 北纬 · 西经", "6": "有效定位 · 南纬 · 西经", "7": "无效定位 · 南纬 · 西经", + }, +}; + +function fieldParts(key) { + const parts = key.split("."); + const category = parts[1] ?? ""; + const suffix = parts.at(-1) ?? key; + const motorMatch = key.match(/\.motors\.motor_(\d+)\./); + const subsystemMatch = key.match(/\.subsystems\.subsystem_(\d+)\./); + const scope = motorMatch ? `电机${motorMatch[1]}-` : subsystemMatch ? `子系统${subsystemMatch[1]}-` : ""; + const categoryName = categoryNames[category] ?? "GB/T 32960 扩展数据"; + const label = fieldNames[suffix]; + if (!label) throw new Error(`missing Chinese field mapping for ${key}`); + const unit = unitBySuffix[suffix] ?? ""; + return { category, categoryName, suffix, label: `${categoryName}-${scope}${label}${unit ? `（${unit}）` : ""}`, unit }; +} + +function displayValue(key, value) { + if (value == null || value === "") return null; + const raw = String(value); + const mapped = valueMappings[key]?.[raw]; + if (mapped) return `${mapped}（${raw}）`; + if (/^0x[0-9a-f]+$/i.test(raw)) return `十六进制 ${raw}`; + if (/^-?\d+(?:\.\d+)?$/.test(raw) && !key.endsWith(".vin")) return Number(raw); + return raw; +} + +function mappingText(key) { + const mapping = valueMappings[key]; + return mapping ? Object.entries(mapping).map(([value, label]) => `${value}=${label}`).join("；") : ""; +} + +function columnName(index) { + let value = index + 1; + let out = ""; + while (value > 0) { + const remainder = (value - 1) % 26; + out = String.fromCharCode(65 + remainder) + out; + value = Math.floor((value - 1) / 26); + } + return out; +} + +const fieldKeys = data.fieldKeys; +const mappedFields = fieldKeys.map((key) => ({ key, ...fieldParts(key) })); +const metadata = [ + ["序号", (item, index) => index + 1], + ["车牌", () => data.query.plate], + ["VIN", (item) => item.vin], + ["协议", (item) => item.protocol], + ["设备时间（北京时间）", (item) => item.event_time], + ["接收时间（北京时间）", (item) => item.received_at], + ["存储时间（北京时间）", (item) => item.ts], + ["消息ID", (item) => item.message_id], + ["消息ID（十六进制）", (item) => item.message_id_hex ? `十六进制 ${item.message_id_hex}` : null], + ["帧ID", (item) => item.frame_id], + ["事件ID", (item) => item.event_id], + ["解析状态", (item) => item.parse_status], + ["解析错误", (item) => item.parse_error ?? null], + ["来源端点", (item) => item.source_endpoint], + ["原始报文字节数", (item) => item.raw_size_bytes], + ["车辆标识", (item) => item.vehicle_key], + ["手机号", (item) => item.phone ?? null], + ["设备ID", (item) => item.device_id ?? null], + ["原始文本", (item) => item.raw_text ?? null], + ["原始报文（十六进制）", (item) => item.raw_hex ?? null], +]; + +const workbook = Workbook.create(); +const summary = workbook.worksheets.add("导出说明"); +const detail = workbook.worksheets.add("完整数据"); +const mapping = workbook.worksheets.add("字段映射"); + +summary.showGridLines = false; +summary.getRange("A1:F1").merge(); +summary.getRange("A1").values = [[`${data.query.plate} · ${exportDate} · GB/T 32960 原始数据导出`]]; +summary.getRange("A1:F1").format = { fill: "#163A5F", font: { bold: true, color: "#FFFFFF", size: 16 }, verticalAlignment: "center" }; +summary.getRange("A1:F1").format.rowHeight = 32; +summary.getRange("A3:B15").values = [ + ["车牌", data.query.plate], + ["VIN", data.query.vin], + ["协议", data.query.protocol], + ["查询日期", `${exportDate}（北京时间）`], + ["查询时间窗", `${data.query.dateFrom} 至 ${data.query.dateTo}`], + ["接口统计总帧数", data.verification.apiTotal], + ["实际导出帧数", data.verification.fetchedRows], + ["唯一帧ID数", data.verification.uniqueFrameIds], + ["唯一事件ID数", data.verification.uniqueEventIds], + ["首条存储时间", data.verification.earliestTs], + ["末条存储时间", data.verification.latestTs], + ["解析字段数", data.verification.parsedFieldCount], + ["消息类型统计", Object.entries(data.verification.messageCounts).map(([key, count]) => `${key}: ${count}帧`).join("；")], +]; +summary.getRange("A3:A15").format = { fill: "#DCE8F2", font: { bold: true, color: "#163A5F" } }; +summary.getRange("A3:B15").format.borders = { preset: "inside", style: "thin", color: "#C8D4E0" }; +summary.getRange("A17:F17").merge(); +summary.getRange("A17").values = [["口径说明"]]; +summary.getRange("A17:F17").format = { fill: "#2B6F91", font: { bold: true, color: "#FFFFFF" } }; +summary.getRange("A18:F22").merge(true); +summary.getRange("A18:F22").values = [ + [`1. 数据源为生产实时 API 的 GB32960 原始帧历史；导出保留原始报文十六进制、帧/事件标识、解析状态及全部 ${data.verification.parsedFieldCount} 个已出现的解析字段。`], + ["2. 完整数据工作表使用中文字段名；字段映射工作表保留原始字段键、单位及状态值映射，便于审计和程序回溯。"], + ["3. 状态类字段显示为“中文含义（协议值）”；普通测量值保持数值，列表/位图/十六进制载荷保持原始文本。"], + [`4. 查询边界为北京时间 ${data.query.dateFrom} 至 ${data.query.dateTo}；该车辆实际存储时间覆盖 ${data.verification.earliestTs} 至 ${data.verification.latestTs}。`], + ["5. 中文术语参考：项目 GB/T 32960 字段展示基准及【字段】32960+广东燃料电池协议.json（SHA-256: 0f0847b86cc692f2130255babbee37e6cb3cf36d717b18d6698164289f66eb82）。"], +]; +summary.getRange("A18:F22").format = { wrapText: true, verticalAlignment: "top", font: { color: "#334155", size: 10 } }; +summary.getRange("A18:F22").format.rowHeight = 34; +summary.getRange("A:A").format.columnWidth = 22; +summary.getRange("B:F").format.columnWidth = 18; +summary.getRange("B:B").format.columnWidth = 78; + +const detailHeaders = [...metadata.map(([label]) => label), ...mappedFields.map((field) => field.label)]; +const detailRows = data.items.map((item, index) => [ + ...metadata.map(([, getter]) => getter(item, index)), + ...mappedFields.map(({ key }) => displayValue(key, item.parsed_fields?.[key])), +]); +const detailMatrix = [detailHeaders, ...detailRows]; +const detailEndColumn = columnName(detailHeaders.length - 1); +detail.getRangeByIndexes(0, 0, detailMatrix.length, detailHeaders.length).values = detailMatrix; +detail.showGridLines = false; +detail.freezePanes.freezeRows(1); +detail.freezePanes.freezeColumns(3); +const dataTable = detail.tables.add(`A1:${detailEndColumn}${detailMatrix.length}`, true, "GB32960DataTable"); +dataTable.style = "TableStyleMedium2"; +dataTable.showFilterButton = true; +dataTable.showBandedColumns = false; +detail.getRangeByIndexes(0, 0, 1, detailHeaders.length).format = { + fill: "#1F4E78", font: { bold: true, color: "#FFFFFF", size: 9 }, wrapText: true, + horizontalAlignment: "center", verticalAlignment: "center", +}; +detail.getRangeByIndexes(0, 0, 1, detailHeaders.length).format.rowHeight = 48; +detail.getRangeByIndexes(1, 0, detailRows.length, detailHeaders.length).format.font = { size: 9, color: "#1F2937" }; +detail.getRangeByIndexes(0, 0, detailMatrix.length, 1).format.columnWidth = 8; +detail.getRangeByIndexes(0, 1, detailMatrix.length, 1).format.columnWidth = 13; +detail.getRangeByIndexes(0, 2, detailMatrix.length, 1).format.columnWidth = 22; +for (const col of [3, 7, 8, 11, 14]) detail.getRangeByIndexes(0, col, detailMatrix.length, 1).format.columnWidth = 14; +for (const col of [4, 5, 6]) detail.getRangeByIndexes(0, col, detailMatrix.length, 1).format.columnWidth = 21; +for (const col of [9, 10]) detail.getRangeByIndexes(0, col, detailMatrix.length, 1).format.columnWidth = 36; +for (const col of [12, 13, 15, 16, 17]) detail.getRangeByIndexes(0, col, detailMatrix.length, 1).format.columnWidth = 20; +detail.getRangeByIndexes(0, 18, detailMatrix.length, 1).format.columnWidth = 28; +detail.getRangeByIndexes(0, 19, detailMatrix.length, 1).format.columnWidth = 44; +for (let col = metadata.length; col < detailHeaders.length; col += 1) { + const key = mappedFields[col - metadata.length].key; + const wide = key.includes("voltages_v") || key.endsWith("faults") || key.endsWith("payload_hex"); + detail.getRangeByIndexes(0, col, detailMatrix.length, 1).format.columnWidth = wide ? 38 : 18; +} +detail.getRangeByIndexes(1, 0, detailRows.length, 1).format.numberFormat = "0"; +detail.getRangeByIndexes(1, 7, detailRows.length, 1).format.numberFormat = "0"; +detail.getRangeByIndexes(1, 14, detailRows.length, 1).format.numberFormat = "0"; + +mapping.showGridLines = false; +mapping.freezePanes.freezeRows(1); +const mappingHeaders = ["序号", "中文字段名", "原始字段键", "数据类别", "单位", "状态值映射", "说明"]; +const mappingRows = mappedFields.map((field, index) => [ + index + 1, + field.label, + field.key, + field.categoryName, + field.unit, + mappingText(field.key), + valueMappings[field.key] ? "完整数据中显示中文含义并保留括号内协议值。" : "值按协议解析结果原样导出；数值字段保持数值类型。", +]); +mapping.getRangeByIndexes(0, 0, mappingRows.length + 1, mappingHeaders.length).values = [mappingHeaders, ...mappingRows]; +const mappingTable = mapping.tables.add(`A1:G${mappingRows.length + 1}`, true, "GB32960FieldMappingTable"); +mappingTable.style = "TableStyleMedium2"; +mapping.getRange("A1:G1").format = { fill: "#1F4E78", font: { bold: true, color: "#FFFFFF" }, wrapText: true, horizontalAlignment: "center" }; +mapping.getRange("A1:G1").format.rowHeight = 30; +mapping.getRange("A:A").format.columnWidth = 8; +mapping.getRange("B:B").format.columnWidth = 42; +mapping.getRange("C:C").format.columnWidth = 58; +mapping.getRange("D:D").format.columnWidth = 30; +mapping.getRange("E:E").format.columnWidth = 12; +mapping.getRange("F:F").format.columnWidth = 68; +mapping.getRange("G:G").format.columnWidth = 48; +mapping.getRange(`B2:G${mappingRows.length + 1}`).format.wrapText = true; + +await fs.mkdir(outputDir, { recursive: true }); + +const summaryInspect = await workbook.inspect({ kind: "table", range: "导出说明!A1:F22", include: "values,formulas", tableMaxRows: 24, tableMaxCols: 8 }); +const detailInspect = await workbook.inspect({ kind: "table", range: "完整数据!A1:Z8", include: "values,formulas", tableMaxRows: 8, tableMaxCols: 26 }); +const mappingInspect = await workbook.inspect({ kind: "table", range: "字段映射!A1:G16", include: "values,formulas", tableMaxRows: 16, tableMaxCols: 8 }); +const errors = await workbook.inspect({ kind: "match", searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", options: { useRegex: true, maxResults: 300 }, summary: "final formula error scan" }); +await fs.writeFile(path.join(taskDir, "summary-inspect.ndjson"), summaryInspect.ndjson, "utf8"); +await fs.writeFile(path.join(taskDir, "detail-inspect.ndjson"), detailInspect.ndjson, "utf8"); +await fs.writeFile(path.join(taskDir, "mapping-inspect.ndjson"), mappingInspect.ndjson, "utf8"); +await fs.writeFile(path.join(taskDir, "formula-errors.ndjson"), errors.ndjson, "utf8"); + +for (const [sheetName, range, filename] of [ + ["导出说明", "A1:F22", "preview-summary.png"], + ["完整数据", "A1:T18", "preview-detail.png"], + ["字段映射", "A1:G28", "preview-mapping.png"], +]) { + const preview = await workbook.render({ sheetName, range, scale: 1.25, format: "png" }); + await fs.writeFile(path.join(taskDir, filename), new Uint8Array(await preview.arrayBuffer())); +} + +const xlsx = await SpreadsheetFile.exportXlsx(workbook); +await xlsx.save(outputPath); +console.log(JSON.stringify({ outputPath, sheets: ["导出说明", "完整数据", "字段映射"], rows: detailRows.length, parsedFields: mappedFields.length, columns: detailHeaders.length }, null, 2)); diff --git a/tmp/gb32960-export-20260817/fetch_data.mjs b/tmp/gb32960-export-20260817/fetch_data.mjs new file mode 100644 index 00000000..f6154677 --- /dev/null +++ b/tmp/gb32960-export-20260817/fetch_data.mjs @@ -0,0 +1,77 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +const baseUrl = process.env.GB32960_BASE_URL || "http://115.29.187.205:20200"; +const vin = process.env.GB32960_VIN || "LB9A32A24R0LS1720"; +const plate = process.env.GB32960_PLATE || "粤AGP4377"; +const exportDate = process.env.GB32960_DATE || "2026-08-17"; +const dateFrom = `${exportDate} 00:00:00`; +const dateTo = `${exportDate} 23:59:59`; +const limit = 500; +const outputPath = process.env.GB32960_DATA_PATH || new URL("./raw-frames.json", import.meta.url).pathname; + +async function fetchPage(offset, includeTotal = false) { + const params = new URLSearchParams({ + protocol: "GB32960", + vin, + dateFrom, + dateTo, + orderBy: "eventTime", + limit: String(limit), + offset: String(offset), + includeFields: "true", + includePayload: "true", + includeTotal: includeTotal ? "true" : "false", + }); + const response = await fetch(`${baseUrl}/api/history/raw-frames?${params}`, { + headers: { "Accept-Encoding": "gzip" }, + signal: AbortSignal.timeout(60000), + }); + if (!response.ok) throw new Error(`HTTP ${response.status}: ${await response.text()}`); + return response.json(); +} + +const first = await fetchPage(0, true); +const total = Number(first.total ?? 0); +const offsets = []; +for (let offset = limit; offset < total; offset += limit) offsets.push(offset); +const remaining = await Promise.all(offsets.map((offset) => fetchPage(offset))); +const items = [...(first.items ?? []), ...remaining.flatMap((page) => page.items ?? [])]; + +const frameIds = new Set(items.map((item) => item.frame_id)); +const eventIds = new Set(items.map((item) => item.event_id)); +const fieldKeys = [...new Set(items.flatMap((item) => Object.keys(item.parsed_fields ?? {})))].sort(); +const messageCounts = Object.fromEntries( + [...items.reduce((map, item) => map.set(item.message_id_hex, (map.get(item.message_id_hex) ?? 0) + 1), new Map())] + .sort(([a], [b]) => a.localeCompare(b)), +); + +if (items.length !== total) throw new Error(`row count mismatch: fetched=${items.length} total=${total}`); +if (frameIds.size !== items.length) throw new Error(`duplicate frame_id: unique=${frameIds.size} rows=${items.length}`); +if (eventIds.size !== items.length) throw new Error(`duplicate event_id: unique=${eventIds.size} rows=${items.length}`); +for (const item of items) { + if (item.protocol !== "GB32960" || item.vin !== vin) throw new Error(`scope mismatch at ${item.frame_id}`); + if (item.ts < dateFrom || item.ts > dateTo) throw new Error(`time out of range at ${item.frame_id}: ${item.ts}`); +} + +const output = { + exportedAt: new Date().toISOString(), + source: `${baseUrl}/api/history/raw-frames`, + query: { plate, vin, protocol: "GB32960", dateFrom, dateTo, orderBy: "eventTime" }, + verification: { + apiTotal: total, + fetchedRows: items.length, + uniqueFrameIds: frameIds.size, + uniqueEventIds: eventIds.size, + earliestTs: items.reduce((min, item) => !min || item.ts < min ? item.ts : min, ""), + latestTs: items.reduce((max, item) => !max || item.ts > max ? item.ts : max, ""), + parsedFieldCount: fieldKeys.length, + messageCounts, + }, + fieldKeys, + items, +}; + +await fs.mkdir(path.dirname(outputPath), { recursive: true }); +await fs.writeFile(outputPath, JSON.stringify(output), "utf8"); +console.log(JSON.stringify({ outputPath, ...output.verification }, null, 2)); diff --git a/tmp/gb32960-export-20260817/verify_final.mjs b/tmp/gb32960-export-20260817/verify_final.mjs new file mode 100644 index 00000000..5609f893 --- /dev/null +++ b/tmp/gb32960-export-20260817/verify_final.mjs @@ -0,0 +1,15 @@ +import fs from "node:fs/promises"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const workbookPath = process.env.GB32960_WORKBOOK_PATH || "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/gb32960-export-20260817-agp4377/粤AGP4377_2026-08-17_GB32960完整数据_中文字段.xlsx"; +const lastDataRow = Number(process.env.GB32960_LAST_ROW || "1233"); +const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(workbookPath)); +const sheets = await workbook.inspect({ kind: "sheet", include: "id,name", maxChars: 3000 }); +const summary = await workbook.inspect({ kind: "table", range: "导出说明!A3:B15", include: "values,formulas", tableMaxRows: 20, tableMaxCols: 3 }); +const detail = await workbook.inspect({ kind: "table", range: "完整数据!A1:T6", include: "values,formulas", tableMaxRows: 6, tableMaxCols: 20 }); +const tail = await workbook.inspect({ kind: "table", range: `完整数据!A${lastDataRow - 4}:F${lastDataRow}`, include: "values,formulas", tableMaxRows: 6, tableMaxCols: 6 }); +const mapping = await workbook.inspect({ kind: "table", range: "字段映射!A1:G10", include: "values,formulas", tableMaxRows: 10, tableMaxCols: 7 }); +const errors = await workbook.inspect({ kind: "match", searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", options: { useRegex: true, maxResults: 300 }, summary: "reopened workbook formula error scan" }); +const output = { sheets: sheets.ndjson, summary: summary.ndjson, detail: detail.ndjson, tail: tail.ndjson, mapping: mapping.ndjson, errors: errors.ndjson }; +await fs.writeFile(new URL("./reopen-verification.json", import.meta.url), JSON.stringify(output, null, 2), "utf8"); +console.log(JSON.stringify({ workbookPath, sizeBytes: (await fs.stat(workbookPath)).size, output }, null, 2)); diff --git a/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/inspect_sources.mjs b/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/inspect_sources.mjs new file mode 100644 index 00000000..08ac08a8 --- /dev/null +++ b/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/inspect_sources.mjs @@ -0,0 +1,94 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const sourceDir = + "/Users/lingniu/Library/Mobile Documents/com~apple~CloudDocs/rsync/2026/07/27"; +const workDir = + "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd"; + +const names = (await fs.readdir(sourceDir)) + .filter((name) => name.endsWith(".xlsx")) + .sort((a, b) => a.localeCompare(b, "zh-CN")); + +const manifest = []; +for (const name of names) { + const filePath = path.join(sourceDir, name); + const input = await FileBlob.load(filePath); + const workbook = await SpreadsheetFile.importXlsx(input); + const sheet = workbook.worksheets.getItemAt(0); + const usedRange = sheet.getUsedRange(true); + const values = usedRange?.values ?? []; + const rowCount = values.length; + const colCount = values.reduce((max, row) => Math.max(max, row.length), 0); + const topRows = values.slice(0, 8).map((row) => + row.slice(0, Math.min(colCount, 40)).map((value) => { + if (value instanceof Date) return value.toISOString(); + return value ?? null; + }), + ); + const preview = await workbook.render({ + sheetName: sheet.name, + range: `A1:${columnName(Math.min(Math.max(colCount, 1), 40))}${Math.min( + Math.max(rowCount, 1), + 15, + )}`, + scale: 1, + format: "png", + }); + const previewName = `${String(manifest.length + 1).padStart(2, "0")}-${safeName( + name, + )}.png`; + await fs.writeFile( + path.join(workDir, previewName), + new Uint8Array(await preview.arrayBuffer()), + ); + manifest.push({ + name, + sheet: sheet.name, + rowCount, + colCount, + firstRow: topRows[0] ?? [], + topRows, + preview: previewName, + }); +} + +await fs.writeFile( + path.join(workDir, "source-manifest.json"), + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8", +); + +console.log( + JSON.stringify( + manifest.map(({ name, sheet, rowCount, colCount, firstRow, preview }) => ({ + name, + sheet, + rowCount, + colCount, + firstRow, + preview, + })), + null, + 2, + ), +); + +function safeName(name) { + return name + .replace(/\.xlsx$/i, "") + .replace(/[^\p{L}\p{N}._-]+/gu, "_") + .slice(0, 100); +} + +function columnName(index) { + let value = index; + let result = ""; + while (value > 0) { + value -= 1; + result = String.fromCharCode(65 + (value % 26)) + result; + value = Math.floor(value / 26); + } + return result || "A"; +} diff --git a/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/normalize_compare.mjs b/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/normalize_compare.mjs new file mode 100644 index 00000000..8a8ae5f8 --- /dev/null +++ b/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd/normalize_compare.mjs @@ -0,0 +1,378 @@ +import fs from "node:fs/promises"; +import fsSync from "node:fs"; +import path from "node:path"; +import readline from "node:readline"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const sourceDir = + "/Users/lingniu/Library/Mobile Documents/com~apple~CloudDocs/rsync/2026/07/27"; +const workDir = + "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd"; +const priorCsv = + "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/g7-mileage-history-20260724/merged/G7车辆每日里程_20220101-20260720.csv"; +const normalizedCsv = path.join( + workDir, + "G7_GPS车辆每日里程_20251231-20260630.csv", +); +const combinedCsv = path.join( + workDir, + "G7_GPS车辆每日里程_合并优先季度统计_20251231-20260630.csv", +); +const statisticCsv = path.join( + workDir, + "G7_GPS季度统计每日里程_20260101-20260630.csv", +); + +const dailyNames = [ + "智能管车_车辆里程日报-091555330.xlsx", + "智能管车_车辆里程日报-090938059.xlsx", + "智能管车_车辆里程日报-091002580.xlsx", + "智能管车_车辆里程日报-091020404.xlsx", + "智能管车_车辆里程日报-091030619.xlsx", + "智能管车_车辆里程日报-091046118.xlsx", + "智能管车_车辆里程日报-091058580.xlsx", +]; +const statisticNames = [ + "里程统计[天][2026-01-01至2026-03-31] (1).xlsx", + "里程统计[天][2026-01-01至2026-03-31].xlsx", + "里程统计[天][2026-04-01至2026-06-30] (1).xlsx", + "里程统计[天][2026-04-01至2026-06-30].xlsx", +]; + +const rows = []; +const newByKey = new Map(); +const dailyFiles = []; +for (const name of dailyNames) { + const values = await readFirstSheetValues(path.join(sourceDir, name)); + const headers = values[0] ?? []; + const runtimeIndex = headers.indexOf("运行时长"); + assert(runtimeIndex > 3, `${name} 缺少运行时长列`); + const dateHeaders = headers.slice(3, runtimeIndex); + const dates = dateHeaders.map((header) => parseHeaderDate(header)); + const seenPlates = new Set(); + let statedTotalKm = 0; + let calculatedTotalKm = 0; + let mismatchRows = 0; + for (const row of values.slice(1)) { + const plate = cleanText(row?.[0]); + if (!plate) continue; + assert(!seenPlates.has(plate), `${name} 存在重复车牌 ${plate}`); + seenPlates.add(plate); + const stated = numberValue(row?.[2], 10_000_000); + const dailyValues = row + .slice(3, runtimeIndex) + .map((value) => numberValue(value)); + const calculated = dailyValues.reduce((sum, value) => sum + value, 0); + statedTotalKm += stated; + calculatedTotalKm += calculated; + if (Math.abs(stated - calculated) > 0.011) mismatchRows += 1; + for (let index = 0; index < dates.length; index += 1) { + const date = dates[index]; + const mileage = dailyValues[index]; + const key = `${plate}|${date}`; + assert(!newByKey.has(key), `日报重复车辆日 ${key}`); + const normalized = { plate, date, mileage }; + newByKey.set(key, normalized); + rows.push(normalized); + } + } + dailyFiles.push({ + name, + vehicleCount: seenPlates.size, + dateFrom: dates[0], + dateTo: dates.at(-1), + dateCount: dates.length, + rowCount: seenPlates.size * dates.length, + statedTotalKm: round(statedTotalKm), + calculatedTotalKm: round(calculatedTotalKm), + mismatchRows, + }); +} + +rows.sort( + (a, b) => + a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date), +); +await fs.writeFile( + normalizedCsv, + `\uFEFFplate,date,daily_mileage_km\n${rows + .map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`) + .join("\n")}\n`, + "utf8", +); + +const statisticByKey = new Map(); +const statisticFiles = []; +for (const name of statisticNames) { + const values = await readFirstSheetValues(path.join(sourceDir, name)); + const headers = (values[1] ?? []).map(cleanText); + const plateIndex = headers.indexOf("车牌号码"); + const methodIndex = headers.indexOf("里程计算方式"); + const dateIndex = headers.indexOf("日期"); + const mileageIndex = headers.indexOf("里程（km）"); + assert( + [plateIndex, methodIndex, dateIndex, mileageIndex].every((index) => index >= 0), + `${name} 表头不完整`, + ); + const fileKeys = new Set(); + const methods = new Set(); + let totalKm = 0; + for (const row of values.slice(2)) { + const plate = cleanText(row?.[plateIndex]); + const date = isoDate(row?.[dateIndex]); + if (!plate || !date) continue; + const method = cleanText(row?.[methodIndex]); + const mileage = numberValue(row?.[mileageIndex]); + const key = `${plate}|${date}`; + assert(!fileKeys.has(key), `${name} 文件内重复车辆日 ${key}`); + assert(!statisticByKey.has(key), `季度统计跨文件重复车辆日 ${key}`); + fileKeys.add(key); + statisticByKey.set(key, { plate, date, mileage, method, name }); + methods.add(method); + totalKm += mileage; + } + statisticFiles.push({ + name, + rowCount: fileKeys.size, + vehicleCount: new Set([...fileKeys].map((key) => key.split("|")[0])).size, + dateCount: new Set([...fileKeys].map((key) => key.split("|")[1])).size, + methods: [...methods].sort(), + totalKm: round(totalKm), + }); +} + +let statisticMatched = 0; +let statisticMissingInDaily = 0; +let statisticChanged = 0; +let statisticAbsoluteDifferenceKm = 0; +let statisticMaxDifferenceKm = 0; +const statisticExamples = []; +for (const [key, statistic] of statisticByKey) { + const daily = newByKey.get(key); + if (!daily) { + statisticMissingInDaily += 1; + continue; + } + statisticMatched += 1; + const difference = daily.mileage - statistic.mileage; + const absolute = Math.abs(difference); + statisticAbsoluteDifferenceKm += absolute; + statisticMaxDifferenceKm = Math.max(statisticMaxDifferenceKm, absolute); + if (absolute > 0.011) { + statisticChanged += 1; + if (statisticExamples.length < 20) { + statisticExamples.push({ + key, + dailyKm: daily.mileage, + statisticKm: statistic.mileage, + differenceKm: round(difference), + }); + } + } +} +const statisticRows = [...statisticByKey.values()].sort( + (a, b) => + a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date), +); +await fs.writeFile( + statisticCsv, + `\uFEFFplate,date,daily_mileage_km\n${statisticRows + .map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`) + .join("\n")}\n`, + "utf8", +); + +const combinedByKey = new Map(newByKey); +for (const [key, statistic] of statisticByKey) { + combinedByKey.set(key, { + plate: statistic.plate, + date: statistic.date, + mileage: statistic.mileage, + }); +} +const combinedRows = [...combinedByKey.values()].sort( + (a, b) => + a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date), +); +await fs.writeFile( + combinedCsv, + `\uFEFFplate,date,daily_mileage_km\n${combinedRows + .map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`) + .join("\n")}\n`, + "utf8", +); + +const priorByKey = await loadPriorRange( + priorCsv, + "2025-12-31", + "2026-06-30", +); +let priorUnchanged = 0; +let priorChanged = 0; +let priorMissing = 0; +let priorExtra = 0; +let priorTotalDifferenceKm = 0; +let priorMaxDifferenceKm = 0; +const priorExamples = []; +for (const [key, row] of newByKey) { + const prior = priorByKey.get(key); + if (!prior) { + priorMissing += 1; + continue; + } + const difference = row.mileage - prior.mileage; + const absolute = Math.abs(difference); + priorTotalDifferenceKm += difference; + priorMaxDifferenceKm = Math.max(priorMaxDifferenceKm, absolute); + if (absolute <= 0.0005) { + priorUnchanged += 1; + } else { + priorChanged += 1; + if (priorExamples.length < 20) { + priorExamples.push({ + key, + newKm: row.mileage, + priorKm: prior.mileage, + differenceKm: round(difference), + }); + } + } +} +for (const key of priorByKey.keys()) { + if (!newByKey.has(key)) priorExtra += 1; +} + +const result = { + normalizedCsv, + combinedCsv, + statisticCsv, + source: { + rowCount: rows.length, + vehicleCount: new Set(rows.map((row) => row.plate)).size, + dateCount: new Set(rows.map((row) => row.date)).size, + dateFrom: rows.reduce( + (min, row) => (!min || row.date < min ? row.date : min), + "", + ), + dateTo: rows.reduce((max, row) => (row.date > max ? row.date : max), ""), + positiveRows: rows.filter((row) => row.mileage > 0).length, + zeroRows: rows.filter((row) => row.mileage === 0).length, + totalKm: round(rows.reduce((sum, row) => sum + row.mileage, 0)), + files: dailyFiles, + }, + quarterlyReconciliation: { + rowCount: statisticByKey.size, + matchedRows: statisticMatched, + missingInDailyRows: statisticMissingInDaily, + changedRowsOver0011Km: statisticChanged, + absoluteDifferenceKm: round(statisticAbsoluteDifferenceKm), + maxDifferenceKm: round(statisticMaxDifferenceKm), + examples: statisticExamples, + files: statisticFiles, + }, + combinedImport: { + rowCount: combinedRows.length, + vehicleCount: new Set(combinedRows.map((row) => row.plate)).size, + dateCount: new Set(combinedRows.map((row) => row.date)).size, + dateFrom: combinedRows.reduce( + (min, row) => (!min || row.date < min ? row.date : min), + "", + ), + dateTo: combinedRows.reduce( + (max, row) => (row.date > max ? row.date : max), + "", + ), + positiveRows: combinedRows.filter((row) => row.mileage > 0).length, + zeroRows: combinedRows.filter((row) => row.mileage === 0).length, + totalKm: round( + combinedRows.reduce((sum, row) => sum + row.mileage, 0), + ), + precedence: + "季度里程统计（明确标注终端里程/经纬度）优先；无季度记录时使用车辆里程日报", + }, + priorImportComparison: { + priorRangeRows: priorByKey.size, + unchangedRows: priorUnchanged, + changedRows: priorChanged, + missingInPriorRows: priorMissing, + extraPriorRows: priorExtra, + totalDifferenceKm: round(priorTotalDifferenceKm), + maxDifferenceKm: round(priorMaxDifferenceKm), + examples: priorExamples, + }, +}; + +await fs.writeFile( + path.join(workDir, "normalization-manifest.json"), + `${JSON.stringify(result, null, 2)}\n`, + "utf8", +); +console.log(JSON.stringify(result, null, 2)); + +async function readFirstSheetValues(file) { + const input = await FileBlob.load(file); + const workbook = await SpreadsheetFile.importXlsx(input); + return workbook.worksheets.getItemAt(0).getUsedRange(true)?.values ?? []; +} + +async function loadPriorRange(file, dateFrom, dateTo) { + const result = new Map(); + const stream = fsSync.createReadStream(file, { encoding: "utf8" }); + const lines = readline.createInterface({ input: stream, crlfDelay: Infinity }); + let first = true; + for await (const line of lines) { + if (first) { + first = false; + continue; + } + if (!line) continue; + const [plate, date, rawMileage] = line.split(","); + if (date < dateFrom || date > dateTo) continue; + result.set(`${plate}|${date}`, { + plate, + date, + mileage: numberValue(rawMileage), + }); + } + return result; +} + +function parseHeaderDate(value) { + const text = cleanText(value); + const match = text.match(/^(\d{2})月(\d{2})日$/); + assert(match, `无法解析日报日期表头 ${text}`); + const month = Number(match[1]); + const year = month === 12 ? 2025 : 2026; + return `${year}-${match[1]}-${match[2]}`; +} + +function isoDate(value) { + if (value instanceof Date) return value.toISOString().slice(0, 10); + const text = cleanText(value); + const match = text.match(/^(\d{4})-(\d{2})-(\d{2})/); + return match ? `${match[1]}-${match[2]}-${match[3]}` : ""; +} + +function cleanText(value) { + return String(value ?? "").trim(); +} + +function numberValue(value, max = 2500) { + if (value === null || value === undefined || value === "") return 0; + const number = Number(String(value).replace(/,/g, "")); + assert(Number.isFinite(number) && number >= 0 && number <= max, `非法里程 ${value}`); + return number; +} + +function formatNumber(value) { + if (Number.isInteger(value)) return String(value); + return String(Number(value.toFixed(3))); +} + +function round(value) { + return Math.round((value + Number.EPSILON) * 1000) / 1000; +} + +function assert(condition, message) { + if (!condition) throw new Error(message); +} diff --git a/tmp/hydrogen-audit-100-real/assess_candidates.mjs b/tmp/hydrogen-audit-100-real/assess_candidates.mjs new file mode 100644 index 00000000..a0655355 --- /dev/null +++ b/tmp/hydrogen-audit-100-real/assess_candidates.mjs @@ -0,0 +1,40 @@ +import fs from 'node:fs/promises'; + +const base = 'http://115.29.187.205:20200'; +const date = '2026-08-25'; +const metrics = JSON.parse(await fs.readFile(new URL('./daily_metrics.json', import.meta.url), 'utf8')); +const candidates = metrics.items + .filter(x => String(x.platform_name || '').startsWith('现代') && Number(x.daily_mileage_km || 0) >= 10) + .sort((a, b) => Number(b.daily_mileage_km) - Number(a.daily_mileage_km)); + +async function one(x) { + const q = new URLSearchParams({ + protocol: 'GB32960', vin: x.vin, + dateFrom: `${date} 00:00:00`, dateTo: `${date} 23:59:59`, + orderBy: 'eventTime', limit: '1', offset: '0', includeFields: 'true', + includePayload: 'false', includeTotal: 'true' + }); + const res = await fetch(`${base}/api/history/raw-frames?${q}`); + if (!res.ok) throw new Error(`${x.vin}: ${res.status}`); + const body = await res.json(); + const f = body.items?.[0]?.parsed_fields || {}; + const required = [ + 'gb32960.fuel_cell.max_hydrogen_pressure_mpa', + 'gb32960.fuel_cell.max_hydrogen_temperature_c', + 'gb32960.vehicle.soc_percent', + 'gb32960.vehicle.total_mileage_km', + 'gb32960.vehicle.vehicle_status', + 'gb32960.vehicle.charge_status' + ]; + return {...x, raw_total: Number(body.total || 0), first_has: required.filter(k => f[k] != null).length}; +} + +const out = []; +for (let i = 0; i < candidates.length; i += 12) { + const batch = candidates.slice(i, i + 12); + out.push(...await Promise.all(batch.map(one))); + process.stderr.write(`checked ${Math.min(i + 12, candidates.length)}\n`); +} +await fs.writeFile(new URL('./candidate_assessment.json', import.meta.url), JSON.stringify(out, null, 2)); +const totals = out.map(x => x.raw_total).sort((a,b)=>a-b); +console.log(JSON.stringify({count: out.length, min: totals[0], median: totals[Math.floor(totals.length/2)], max: totals.at(-1), complete_first_frame: out.filter(x=>x.first_has===6).length, prefixes: Object.entries(Object.groupBy(out, x=>x.vin.slice(0,3))).map(([prefix, rows]) => ({prefix, count: rows.length}))}, null, 2)); diff --git a/tmp/hydrogen-audit-100-real/build_real_audit.mjs b/tmp/hydrogen-audit-100-real/build_real_audit.mjs new file mode 100644 index 00000000..4e0373ef --- /dev/null +++ b/tmp/hydrogen-audit-100-real/build_real_audit.mjs @@ -0,0 +1,95 @@ +import fs from "node:fs/promises"; +import { Workbook, SpreadsheetFile } from "@oai/artifact-tool"; + +const root="/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest"; +const inputDir=`${root}/tmp/hydrogen-audit-100-real/output-v4`; +const outputRoot=`${root}/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083`; +const packageDir=`${outputRoot}/100个真实车辆日氢耗核验资料包_V3`; +const outputPath=`${packageDir}/100个真实车辆日氢耗人工验算底稿_V3.xlsx`; +const previewDir=`${outputRoot}/preview-real-100-v3`; +const days=JSON.parse(await fs.readFile(`${inputDir}/daily_results.json`,"utf8")); +const C={navy:"#0B3A53",blue:"#0E7490",cyan:"#CDEEF4",pale:"#EAF6F8",white:"#FFFFFF",green:"#DCFCE7",greenText:"#166534",orange:"#FFEDD5",orangeText:"#9A3412",red:"#FEE2E2",redText:"#991B1B"}; +const coeffA=[0.05888460,-0.06136111,-0.002650473,0.002731125,0.001802374,-0.001150707,0.00009588528,-0.0000001109040,0.0000000001264403],coeffB=[1.325,1.87,2.5,2.8,2.938,3.14,3.37,3.75,4.0],coeffC=[1,1,2,2,2.42,2.63,3,4,5]; +function col(n){let s="";while(n){n--;s=String.fromCharCode(65+n%26)+s;n=Math.floor(n/26)}return s} +function title(sheet,address,text){const r=sheet.getRange(address);r.merge();r.values=[[text]];r.format={fill:C.navy,font:{bold:true,color:C.white,size:16},verticalAlignment:"center",horizontalAlignment:"left"};r.format.rowHeight=34} +function header(r){r.format={fill:C.blue,font:{bold:true,color:C.white},verticalAlignment:"center",horizontalAlignment:"center",wrapText:true,borders:{preset:"all",style:"thin",color:"#B8CDD5"}};r.format.rowHeight=34} +function body(r){r.format={verticalAlignment:"center",borders:{insideHorizontal:{style:"thin",color:"#DDE7EB"},bottom:{style:"thin",color:"#B8CDD5"}}}} +function widths(sheet,arr){arr.forEach((w,i)=>sheet.getRange(`${col(i+1)}:${col(i+1)}`).format.columnWidth=w)} +function val(v){return v===undefined||v===null?null:v} +function zFormula(p,t){return ["1",...Array.from({length:9},(_,i)=>`'参数与公式'!$B$${26+i}*POWER(100/(${t}+273.15),'参数与公式'!$C$${26+i})*POWER(${p},'参数与公式'!$D$${26+i})`)].join("+")} +function massFormula(p,t,v){return `${p}*1000*0.00201588*${v}/(8.314472*(${t}+273.15)*(${zFormula(p,t)}))`} + +const wb=Workbook.create();wb.comments.setSelf({displayName:"车辆数据中台"}); +const guide=wb.worksheets.add("核验说明"),params=wb.worksheets.add("参数与公式"),summary=wb.worksheets.add("100车辆日结果"),h2Detail=wb.worksheets.add("氢量计算明细"),operation=wb.worksheets.add("运行周期明细"),rawIndex=wb.worksheets.add("原始帧索引"),fields=wb.worksheets.add("原始字段说明"); +for(const s of [guide,params,summary,h2Detail,operation,rawIndex,fields])s.showGridLines=false; + +title(guide,"A1:H1","100个真实车辆日氢耗人工验算底稿（V3）");guide.getRange("A3:B3").values=[["项目","说明"]];header(guide.getRange("A3:B3")); +const statusCounts=Object.fromEntries(["OK","SUSPECT","NO_DATA"].map(k=>[k,days.filter(d=>d.stat.QualityStatus===k).length])); +const guideRows=[ + ["样本口径","2026-08-25，固定100个真实车辆日；4.5T普货41辆（380 L）、4.5T冷链59辆（520 L），动力电池均为21.04 kWh。"], + ["原始数据","沿用原100个样本的232,291条GB/T 32960原始帧，未更换车辆、日期或报文；每车1个CSV.GZ，保留原始HEX及中文映射字段。"], + ["有效分界点","取消上下电前后60秒窗口和首尾中位数，直接取满足车辆启动、字段完整、压力温度有效且不在外部充电/异常压力记录中的有效分界点。"], + ["物理用氢量","无加氢：首个有效运行分界点剩余氢量－最后有效运行分界点剩余氢量；有加氢：按加氢事件切段后累计各段氢量下降。充电不切分物理氢量。"], + ["充电识别","仅当充电状态=1且车辆状态=2时识别为外部充电；每次充电完成后建立新充电周期。"], + ["纯电里程","每个充电周期若启动时为纯电，仅累计首次连续纯电前缀；混动连续3帧且相邻不超过30秒后，从首帧分界点锁定为混动，直至下一次充电。若周期从混动启动，则纯电里程为0。"], + ["跨日状态","日界线只用于出报，不重置充电周期。若缺少日初已持久化状态，按已进入混动处理，不虚增日初纯电里程，并标记SUSPECT供复核。"], + ["SOC修正","只使用各充电周期混动段的状态边界：ΔSOC=末SOC－初SOC；电等效氢=电池容量×ΔSOC÷100÷16。SOC上升为正并从用氢量扣除；SOC下降为负，扣除负值即补回电池放电能量。"], + ["最终公式","修正耗氢量=物理用氢量－用电等效氢；混动里程=总里程－纯电里程；修正百公里氢耗=修正耗氢量÷混动里程×100。"], + ["人工核对顺序","①在“100车辆日结果”看系统值与Excel复算差；②在“氢量计算明细”核对加氢分段及压力质量；③在“运行周期明细”核对纯电前缀和混动SOC；④按事件ID回查原始CSV。"], + ["重算结果",`OK ${statusCounts.OK}辆日；SUSPECT ${statusCounts.SUSPECT}辆日；NO_DATA ${statusCounts.NO_DATA}辆日。SUSPECT不删除结果，保留原因和原始证据供人工判断。`], + ["算法版本","PRESSURE_NIST_VALID_BOUNDARY_CHARGE_CYCLE_V3"]]; +guide.getRange(`A4:B${3+guideRows.length}`).values=guideRows;body(guide.getRange(`A4:B${3+guideRows.length}`));guide.getRange(`B4:B${3+guideRows.length}`).format.wrapText=true;widths(guide,[24,112]);guide.freezePanes.freezeRows(3); + +title(params,"A1:F1","系统参数、压力—质量公式与业务计算口径");params.getRange("A3:C3").values=[["参数","取值","说明"]];header(params.getRange("A3:C3")); +const pRows=[["4.5T普货储氢总容积(L)",380,"中台VIN参数表"],["4.5T冷链储氢总容积(L)",520,"中台VIN参数表"],["动力电池容量(kWh)",21.04,"两类4.5T车辆一致"],["氢气等效电量(kWh/kg)",16,"SOC能量折算"],["上电/下电时间窗口(s)",0,"直接使用有效分界点"],["加氢压升阈值(MPa)",3,"需持续300秒"],["加氢持续时间(s)",300,"5分钟"],["纯电异常压降(MPa)",5,"30分钟窗口"],["纯电异常窗口(s)",1800,"30分钟"],["混动确认连续帧数",3,"避免单帧运行模式抖动"],["混动确认相邻最大间隔(s)",30,"确认后回溯到首个混动帧"],["最低混动里程(km)",10,"不足标记SUSPECT"]]; +params.getRange(`A4:C${3+pRows.length}`).values=pRows;body(params.getRange(`A4:C${3+pRows.length}`));params.getRange(`B4:B${3+pRows.length}`).format.numberFormat="0.000"; +params.getRange("A18:F18").merge();params.getRange("A18:F18").values=[["核心公式"]];params.getRange("A18:F18").format={fill:C.cyan,font:{bold:true,color:C.navy}}; +const formulas=["Z = 1 + Σ[aᵢ × (100/Tₖ)^bᵢ × P^cᵢ]","剩余氢量 m = P × 1000 × 0.00201588 × V ÷ (8.314472 × Tₖ × Z)","ΔSOC = SOC_end − SOC_start；用电等效氢 = 电池容量 × ΔSOC ÷ 100 ÷ 16","修正耗氢量 = 物理用氢量 − 用电等效氢","混动里程 = 总里程 − 纯电里程；百公里氢耗 = 修正耗氢量 ÷ 混动里程 × 100"]; +for(let i=0;i[i+1,a,coeffB[i],coeffC[i]]);body(params.getRange("A26:D34"));params.getRange("B26:D34").format.numberFormat="0.0000000000"; +wb.comments.addThread({cell:params.getRange("B4")},"普货按中台VIN车型参数取380 L，不与冷链520 L混用。");wb.comments.addThread({cell:params.getRange("B5")},"冷链按中台VIN车型参数取520 L。");wb.comments.addThread({cell:params.getRange("B6")},"4.5T普货和冷链动力电池容量均为21.04 kWh。");wb.comments.addThread({cell:params.getRange("B7")},"业务确认：1 kg氢气等效16 kWh，用于SOC能量折算。");widths(params,[32,18,58,15,15,15]);params.freezePanes.freezeRows(3); + +const hHeaders=["样例","车牌","日期","VIN","车型","氢量段号","分段说明","开始时间","结束时间","开始事件ID","结束事件ID","数据源","储氢容积(L)","起始压力(MPa)","结束压力(MPa)","起始温度(℃)","结束温度(℃)","系统起始氢量(kg)","系统结束氢量(kg)","系统分段用氢(kg)","起始里程(km)","结束里程(km)","区间里程(km)","样本数","质量状态","起始氢量复算","结束氢量复算","起始质量差","结束质量差","分段用氢复算","用氢差值","复算结论"]; +h2Detail.getRange(`A1:${col(hHeaders.length)}1`).values=[hHeaders];header(h2Detail.getRange(`A1:${col(hHeaders.length)}1`));const hRows=[];for(const d of days)for(const it of d.stat.HydrogenIntervals||[])hRows.push([d.candidate.sample_no,d.candidate.plate,d.candidate.stat_date,d.candidate.vin,d.candidate.model,it.index,it.qualityReason,it.startTime,it.endTime,it.startEventId,it.endEventId,it.source,d.candidate.tank_capacity_l,it.startPressureMpa,it.endPressureMpa,it.startTemperatureC,it.endTemperatureC,it.startMassKg,it.endMassKg,it.rawHydrogenConsumptionKg,val(it.startMileageKm),val(it.endMileageKm),val(it.mileageKm),it.sampleCount,it.qualityStatus]); +h2Detail.getRange(`A2:Y${hRows.length+1}`).values=hRows;for(let r=2;r<=hRows.length+1;r++){h2Detail.getRange(`Z${r}`).formulas=[[`=${massFormula(`N${r}`,`P${r}`,`M${r}`)}`]];h2Detail.getRange(`AA${r}`).formulas=[[`=${massFormula(`O${r}`,`Q${r}`,`M${r}`)}`]];h2Detail.getRange(`AB${r}`).formulas=[[`=Z${r}-R${r}`]];h2Detail.getRange(`AC${r}`).formulas=[[`=AA${r}-S${r}`]];h2Detail.getRange(`AD${r}`).formulas=[[`=Z${r}-AA${r}`]];h2Detail.getRange(`AE${r}`).formulas=[[`=AD${r}-T${r}`]];h2Detail.getRange(`AF${r}`).formulas=[[`=IF(MAX(ABS(AB${r}),ABS(AC${r}),ABS(AE${r}))<=0.002,"一致","复核")`]]} +body(h2Detail.getRange(`A2:AF${hRows.length+1}`));h2Detail.getRange(`M2:AE${hRows.length+1}`).format.numberFormat="0.000000";h2Detail.getRange(`Y2:Y${hRows.length+1}`).conditionalFormats.add("containsText",{text:"SUSPECT",format:{fill:C.orange,font:{color:C.orangeText,bold:true}}});h2Detail.getRange(`AF2:AF${hRows.length+1}`).conditionalFormats.add("containsText",{text:"一致",format:{fill:C.green,font:{color:C.greenText,bold:true}}});h2Detail.getRange(`AF2:AF${hRows.length+1}`).conditionalFormats.add("containsText",{text:"复核",format:{fill:C.red,font:{color:C.redText,bold:true}}});h2Detail.tables.add(`A1:AF${hRows.length+1}`,true,"HydrogenEvidenceV3").style="TableStyleMedium2";h2Detail.freezePanes.freezeRows(1);h2Detail.freezePanes.freezeColumns(7);widths(h2Detail,[8,13,13,22,24,10,34,22,22,34,34,24,14,14,14,14,14,17,17,17,15,15,15,10,13,17,17,14,14,17,14,12]); + +const oHeaders=["样例","车牌","日期","VIN","车型","充电周期号","区间类型","区间号","开始时间","结束时间","开始事件ID","结束事件ID","数据源","储氢容积(L)","电池容量(kWh)","起始压力(MPa)","结束压力(MPa)","起始温度(℃)","结束温度(℃)","系统起始氢量","系统结束氢量","区间物理氢量","起始SOC","结束SOC","系统ΔSOC(末-初)","系统电量变化(kWh)","系统电等效氢(kg)","系统段修正氢量","起始里程","结束里程","系统区间里程","系统段百公里","样本数","质量状态","分界说明","起始氢量复算","结束氢量复算","起始质量差","结束质量差","ΔSOC复算","电量变化复算","电等效氢复算","段修正氢量复算","里程复算","纯电里程贡献","混动电等效氢贡献","复算结论"]; +operation.getRange(`A1:${col(oHeaders.length)}1`).values=[oHeaders];header(operation.getRange(`A1:${col(oHeaders.length)}1`));const oRows=[];for(const d of days){let cycleNo=0,previous="";for(const it of d.stat.Intervals||[]){if(cycleNo===0||it.type==="PURE_ELECTRIC"||previous==="MIXED")cycleNo++;oRows.push([d.candidate.sample_no,d.candidate.plate,d.candidate.stat_date,d.candidate.vin,d.candidate.model,cycleNo,it.type,it.index,it.startTime,it.endTime,it.startEventId,it.endEventId,it.source,d.candidate.tank_capacity_l,21.04,it.startPressureMpa,it.endPressureMpa,it.startTemperatureC,it.endTemperatureC,it.startMassKg,it.endMassKg,it.rawHydrogenConsumptionKg,val(it.startSocPercent),val(it.endSocPercent),val(it.batterySocDeltaPct),val(it.batteryEnergyChangeKWh),val(it.electricEquivalentHydrogenKg),val(it.correctedHydrogenConsumptionKg),val(it.startMileageKm),val(it.endMileageKm),val(it.mileageKm),val(it.consumptionKgPer100Km),it.sampleCount,it.qualityStatus,it.qualityReason]);previous=it.type}} +operation.getRange(`A2:AI${oRows.length+1}`).values=oRows;for(let r=2;r<=oRows.length+1;r++){operation.getRange(`AJ${r}`).formulas=[[`=${massFormula(`P${r}`,`R${r}`,`N${r}`)}`]];operation.getRange(`AK${r}`).formulas=[[`=${massFormula(`Q${r}`,`S${r}`,`N${r}`)}`]];operation.getRange(`AL${r}`).formulas=[[`=AJ${r}-T${r}`]];operation.getRange(`AM${r}`).formulas=[[`=AK${r}-U${r}`]];operation.getRange(`AN${r}`).formulas=[[`=IF(OR(W${r}="",X${r}=""),"",X${r}-W${r})`]];operation.getRange(`AO${r}`).formulas=[[`=IF(AN${r}="","",O${r}*AN${r}/100)`]];operation.getRange(`AP${r}`).formulas=[[`=IF(AO${r}="","",AO${r}/'参数与公式'!$B$7)`]];operation.getRange(`AQ${r}`).formulas=[[`=IF(G${r}="MIXED",V${r}-AP${r},"")`]];operation.getRange(`AR${r}`).formulas=[[`=IF(OR(AC${r}="",AD${r}=""),"",AD${r}-AC${r})`]];operation.getRange(`AS${r}`).formulas=[[`=IF(G${r}="PURE_ELECTRIC",AR${r},0)`]];operation.getRange(`AT${r}`).formulas=[[`=IF(G${r}="MIXED",AP${r},0)`]];operation.getRange(`AU${r}`).formulas=[[`=IF(MAX(ABS(AL${r}),ABS(AM${r}),ABS(AN${r}-Y${r}),ABS(AP${r}-AA${r}),ABS(AR${r}-AE${r}))<=0.002,"一致","复核")`]]} +body(operation.getRange(`A2:AU${oRows.length+1}`));operation.getRange(`N2:AT${oRows.length+1}`).format.numberFormat="0.000000";operation.getRange(`G2:G${oRows.length+1}`).conditionalFormats.add("containsText",{text:"PURE_ELECTRIC",format:{fill:C.cyan,font:{color:C.navy}}});operation.getRange(`G2:G${oRows.length+1}`).conditionalFormats.add("containsText",{text:"MIXED",format:{fill:C.green,font:{color:C.greenText}}});operation.getRange(`AU2:AU${oRows.length+1}`).conditionalFormats.add("containsText",{text:"一致",format:{fill:C.green,font:{color:C.greenText,bold:true}}});operation.getRange(`AU2:AU${oRows.length+1}`).conditionalFormats.add("containsText",{text:"复核",format:{fill:C.red,font:{color:C.redText,bold:true}}});operation.tables.add(`A1:AU${oRows.length+1}`,true,"OperationCyclesV3").style="TableStyleMedium2";operation.freezePanes.freezeRows(1);operation.freezePanes.freezeColumns(8);widths(operation,[8,13,13,22,24,11,15,9,22,22,34,34,24,14,15,14,14,14,14,16,16,16,12,12,15,17,17,18,15,15,16,17,10,13,42,17,17,14,14,15,17,17,18,15,16,20,12]); + +const sHeaders=["样例","日期","车牌","VIN","车型","储氢容积(L)","电池容量(kWh)","原统计日里程(km)","有效起始仪表里程","有效结束仪表里程","总里程复算","API原始帧","算法样本","氢量分段数","运行区间数","加氢次数","充电次数","异常记录","系统物理用氢","物理用氢复算","物理差值","系统ΔSOC","ΔSOC复算","系统电等效氢","电等效氢复算","等效氢差值","系统修正耗氢","修正耗氢复算","修正差值","系统纯电里程","纯电里程复算","系统混动里程","混动里程复算","混动差值","系统物理百公里","物理百公里复算","系统修正百公里","修正百公里复算","百公里差值","质量状态","质量原因","算法版本","原始文件","打开原始帧"]; +summary.getRange(`A1:${col(sHeaders.length)}1`).values=[sHeaders];header(summary.getRange(`A1:${col(sHeaders.length)}1`)); +const sRows=days.map(d=>{const s=d.stat,c=d.candidate;return[c.sample_no,c.stat_date,c.plate,c.vin,c.model,c.tank_capacity_l,21.04,c.daily_mileage_km,val(s.FirstObservation?.MileageKm),val(s.LastObservation?.MileageKm),null,d.apiRawFrameCount,d.algorithmSamples,(s.HydrogenIntervals||[]).length,(s.Intervals||[]).length,s.RefuelCount,s.ChargeCount,s.InvalidSegmentCount+s.AbnormalDropCount,s.ConsumptionKg,null,null,val(s.BatterySOCDeltaPct),null,val(s.ElectricEquivalentKg),null,null,val(s.CorrectedConsumptionKg),null,null,s.PureElectricMileageKm,null,s.MixedMileageKm,null,null,val(s.ConsumptionKgPer100Km),null,val(s.SOCBalancedKgPer100Km),null,null,s.QualityStatus,s.QualityReason,s.CalculationParameters?.algorithmVersion,d.rawArchiveFile,null]}); +summary.getRange(`A2:AR${sRows.length+1}`).values=sRows;const hEnd=hRows.length+1,oEnd=oRows.length+1; +for(let r=2;r<=sRows.length+1;r++){ + summary.getRange(`K${r}`).formulas=[[`=IF(OR(I${r}="",J${r}=""),"",J${r}-I${r})`]]; + summary.getRange(`T${r}`).formulas=[[`=SUMIFS('氢量计算明细'!$AD$2:$AD$${hEnd},'氢量计算明细'!$D$2:$D$${hEnd},$D${r})`]];summary.getRange(`U${r}`).formulas=[[`=T${r}-S${r}`]]; + summary.getRange(`W${r}`).formulas=[[`=SUMIFS('运行周期明细'!$AN$2:$AN$${oEnd},'运行周期明细'!$D$2:$D$${oEnd},$D${r},'运行周期明细'!$G$2:$G$${oEnd},"MIXED")`]]; + summary.getRange(`Y${r}`).formulas=[[`=SUMIFS('运行周期明细'!$AT$2:$AT$${oEnd},'运行周期明细'!$D$2:$D$${oEnd},$D${r})`]];summary.getRange(`Z${r}`).formulas=[[`=Y${r}-X${r}`]]; + summary.getRange(`AB${r}`).formulas=[[`=T${r}-Y${r}`]];summary.getRange(`AC${r}`).formulas=[[`=AB${r}-AA${r}`]]; + summary.getRange(`AE${r}`).formulas=[[`=SUMIFS('运行周期明细'!$AS$2:$AS$${oEnd},'运行周期明细'!$D$2:$D$${oEnd},$D${r})`]]; + summary.getRange(`AG${r}`).formulas=[[`=K${r}-AE${r}`]];summary.getRange(`AH${r}`).formulas=[[`=AG${r}-AF${r}`]]; + summary.getRange(`AJ${r}`).formulas=[[`=IF(AG${r}>0,T${r}*100/AG${r},"")`]];summary.getRange(`AL${r}`).formulas=[[`=IF(AG${r}>0,AB${r}*100/AG${r},"")`]];summary.getRange(`AM${r}`).formulas=[[`=IF(OR(AK${r}="",AL${r}=""),"",AL${r}-AK${r})`]]; + summary.getRange(`AR${r}`).formulas=[[`=HYPERLINK("${days[r-2].rawArchiveFile}","打开")`]]; +} +body(summary.getRange(`A2:AR${sRows.length+1}`));summary.getRange(`F2:AM${sRows.length+1}`).format.numberFormat="0.000000"; +for(const [text,fill,font] of [["OK",C.green,C.greenText],["SUSPECT",C.orange,C.orangeText],["NO_DATA",C.red,C.redText]])summary.getRange(`AN2:AN${sRows.length+1}`).conditionalFormats.add("containsText",{text,format:{fill,font:{color:font,bold:true}}}); +for(const c of ["U","Z","AC","AH","AM"]){summary.getRange(`${c}2:${c}${sRows.length+1}`).conditionalFormats.add("cellIs",{operator:"greaterThan",formula:0.002,format:{fill:C.red,font:{color:C.redText,bold:true}}});summary.getRange(`${c}2:${c}${sRows.length+1}`).conditionalFormats.add("cellIs",{operator:"lessThan",formula:-0.002,format:{fill:C.red,font:{color:C.redText,bold:true}}})} +summary.tables.add(`A1:AR${sRows.length+1}`,true,"VehicleDayResultsV3").style="TableStyleMedium2";summary.freezePanes.freezeRows(1);summary.freezePanes.freezeColumns(5);widths(summary,[8,13,13,22,24,14,14,17,18,18,15,12,12,12,12,11,11,12,16,16,14,14,14,16,16,14,17,17,14,16,16,16,16,14,18,18,18,18,14,13,48,42,58,12]); + +const rHeaders=["样例","车牌","VIN","日期","车型","储氢容积(L)","电池容量(kWh)","API原始帧数","唯一帧数","重复帧数","算法样本数","关键字段帧数","原始最早事件时间","原始最晚事件时间","原始文件","打开"]; +rawIndex.getRange("A1:P1").values=[rHeaders];header(rawIndex.getRange("A1:P1"));const rRows=days.map(d=>[d.candidate.sample_no,d.candidate.plate,d.candidate.vin,d.candidate.stat_date,d.candidate.model,d.candidate.tank_capacity_l,21.04,d.apiRawFrameCount,d.uniqueFrameCount,d.duplicateFrameCount,d.algorithmSamples,d.criticalFieldRows,d.earliestEventTime,d.latestEventTime,d.rawArchiveFile,null]);rawIndex.getRange(`A2:P${rRows.length+1}`).values=rRows;for(let r=2;r<=rRows.length+1;r++)rawIndex.getRange(`P${r}`).formulas=[[`=HYPERLINK("${days[r-2].rawArchiveFile}","打开")`]];body(rawIndex.getRange(`A2:P${rRows.length+1}`));rawIndex.tables.add(`A1:P${rRows.length+1}`,true,"RawFrameManifestV3").style="TableStyleMedium2";rawIndex.freezePanes.freezeRows(1);rawIndex.freezePanes.freezeColumns(5);widths(rawIndex,[8,13,22,13,24,14,14,14,12,12,14,15,21,21,58,12]); + +title(fields,"A1:D1","原始帧字段及人工核验用途");fields.getRange("A3:D3").values=[["中文字段","GB/T 32960解析键/原始属性","用途","校验规则/枚举"]];header(fields.getRange("A3:D3")); +const fRows=[["原始报文HEX","raw_hex","回查原始帧","完整保留，不参与公式"],["事件ID","event_id","连接各明细起止事件","按ID在对应CSV查找"],["事件时间","event_time","确定自然日及排序","以事件时间为准"],["最高氢压(MPa)","gb32960.fuel_cell.max_hydrogen_pressure_mpa","压力—质量换算","0-40"],["电池SOC(%)","gb32960.vehicle.soc_percent","混动边界SOC修正","0–100；只取相关状态边界"],["仪表总里程(km)","gb32960.vehicle.total_mileage_km","总/纯电/混动里程","分界点末值-初值"],["车辆状态","gb32960.vehicle.vehicle_status","启动及充电联合判断","1=启动，2=熄火，其他按国标保留"],["充电状态","gb32960.vehicle.charge_status","外部充电联合判断","1=停车充电，2=行驶充电，3=未充电，4=充电完成"],["运行模式","gb32960.vehicle.running_mode","纯电前缀/混动锁定","1=纯电，2=混动，3=燃油，254=异常，255=无效"],["燃料电池工作状态","gb32960.gd_fc_stack.engine_work_state","辅助确认纯电/混动","2=工作；0/1=未工作/停机"],["系统压力换算质量(kg)","工作底稿复算列","氢量端点","按VIN容积+NIST公式"],["计算角色/排除原因","工作底稿标注","定位参与/排除状态","有效分界点、外部充电、异常或字段缺失"]]; +fields.getRange(`A4:D${3+fRows.length}`).values=fRows;body(fields.getRange(`A4:D${3+fRows.length}`));fields.getRange(`A4:D${3+fRows.length}`).format.wrapText=true;widths(fields,[28,62,40,54]);fields.freezePanes.freezeRows(3); + +await fs.mkdir(packageDir,{recursive:true}); +const checks={};for(const [name,range] of [["核验说明","A1:H15"],["参数与公式","A1:F34"],["100车辆日结果","A1:AR12"],["氢量计算明细","A1:AF15"],["运行周期明细","A1:AU15"],["原始帧索引","A1:P15"],["原始字段说明","A1:D16"]])checks[name]=(await wb.inspect({kind:"table",sheetId:name,range,include:"values,formulas",tableMaxRows:18,tableMaxCols:50,maxChars:18000})).ndjson; +const formulaErrors=await wb.inspect({kind:"match",searchTerm:"#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A",options:{useRegex:true,maxResults:300},summary:"final formula error scan",maxChars:10000}); +const xlsx=await SpreadsheetFile.exportXlsx(wb);await xlsx.save(outputPath); +await fs.rm(previewDir,{recursive:true,force:true});await fs.mkdir(previewDir,{recursive:true});const renderRanges={"核验说明":"A1:H15","参数与公式":"A1:F34","100车辆日结果":"A1:AR14","氢量计算明细":"A1:AF14","运行周期明细":"A1:AU14","原始帧索引":"A1:P14","原始字段说明":"A1:D16"};for(const [name,range] of Object.entries(renderRanges)){const image=await wb.render({sheetName:name,range,scale:1,format:"png"});await fs.writeFile(`${previewDir}/${name}.png`,new Uint8Array(await image.arrayBuffer()))} +await fs.cp(`${inputDir}/raw`,`${packageDir}/raw`,{recursive:true,force:true});for(const name of ["daily_results.json","raw_manifest.csv","selected_vehicles.json"])await fs.copyFile(`${inputDir}/${name}`,`${packageDir}/${name}`); +await fs.writeFile(`${outputRoot}/verification-real-100-v3.json`,JSON.stringify({outputPath,checks,formulaErrors:formulaErrors.ndjson,days:days.length,hydrogenIntervals:hRows.length,operationIntervals:oRows.length},null,2));console.log(JSON.stringify({outputPath,days:days.length,hydrogenIntervals:hRows.length,operationIntervals:oRows.length,previewDir,formulaErrors:formulaErrors.ndjson},null,2)); diff --git a/tmp/hydrogen-audit-100-real/inspect_v3.mjs b/tmp/hydrogen-audit-100-real/inspect_v3.mjs new file mode 100644 index 00000000..15e4e029 --- /dev/null +++ b/tmp/hydrogen-audit-100-real/inspect_v3.mjs @@ -0,0 +1,10 @@ +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const path = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083/100个真实车辆日氢耗核验资料包_V3/100个真实车辆日氢耗人工验算底稿_V3.xlsx"; +const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(path)); +for (const [sheetId, range] of [["100车辆日结果", "A53:AR53"], ["氢量计算明细", "A1:AF4"], ["运行周期明细", "A1:AU8"]]) { + const result = await workbook.inspect({kind:"table", sheetId, range, include:"values,formulas", tableMaxRows:10, tableMaxCols:50, maxChars:16000}); + console.log(result.ndjson); +} +const errors = await workbook.inspect({kind:"match",searchTerm:"#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A",options:{useRegex:true,maxResults:300},summary:"final formula error scan",maxChars:4000}); +console.log(errors.ndjson); diff --git a/tmp/hydrogen-formula-analysis-20260826/inspect.mjs b/tmp/hydrogen-formula-analysis-20260826/inspect.mjs new file mode 100644 index 00000000..48093330 --- /dev/null +++ b/tmp/hydrogen-formula-analysis-20260826/inspect.mjs @@ -0,0 +1,46 @@ +import fs from "node:fs/promises"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const inputPath = "/Users/lingniu/Downloads/氢气计算公式.xlsx"; +const outputDir = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/hydrogen-formula-analysis-20260826/preview"; +await fs.mkdir(outputDir, { recursive: true }); + +const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(inputPath)); +const overview = await workbook.inspect({ + kind: "workbook,sheet,table", + include: "id,name,values,formulas", + maxChars: 12000, + tableMaxRows: 30, + tableMaxCols: 20, + tableMaxCellChars: 300, +}); +console.log("OVERVIEW"); +console.log(overview.ndjson); + +const sheets = (await workbook.inspect({ kind: "sheet", include: "id,name", maxChars: 6000 })).ndjson + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)) + .filter((item) => item.kind === "sheet"); + +for (let index = 0; index < sheets.length; index += 1) { + const sheet = sheets[index]; + const region = await workbook.inspect({ + kind: "region,formula,computedStyle", + sheetId: sheet.name, + range: "A1:Z80", + include: "values,formulas", + maxChars: 30000, + tableMaxRows: 80, + tableMaxCols: 26, + tableMaxCellChars: 500, + options: { maxResults: 300 }, + }); + console.log(`SHEET ${sheet.name}`); + console.log(region.ndjson); + const rendered = await workbook.render({ sheetName: sheet.name, autoCrop: "all", scale: 2, format: "png" }); + const safeName = String(sheet.name).replace(/[\\/:*?"<>|]/g, "_"); + await fs.writeFile(`${outputDir}/${String(index + 1).padStart(2, "0")}-${safeName}.png`, new Uint8Array(await rendered.arrayBuffer())); +} + +console.log(JSON.stringify({ inputPath, outputDir, sheetCount: sheets.length })); diff --git a/tmp/hydrogen_audit_120/build_hydrogen_audit.mjs b/tmp/hydrogen_audit_120/build_hydrogen_audit.mjs new file mode 100644 index 00000000..2d0d8a8f --- /dev/null +++ b/tmp/hydrogen_audit_120/build_hydrogen_audit.mjs @@ -0,0 +1,496 @@ +import fs from "node:fs/promises"; +import { SpreadsheetFile, Workbook } from "@oai/artifact-tool"; + +const outputDir = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083"; +const outputPath = `${outputDir}/氢耗算法120组人工验算底稿.xlsx`; +const previewDir = `${outputDir}/preview`; + +const colors = { + navy: "#0B3A53", + blue: "#0E7490", + cyan: "#CDEEF4", + pale: "#EAF6F8", + green: "#DCFCE7", + greenText: "#166534", + orange: "#FFEDD5", + orangeText: "#9A3412", + red: "#FEE2E2", + redText: "#991B1B", + gray: "#E5E7EB", + grayText: "#475569", + white: "#FFFFFF", +}; + +const coeffA = [0.05888460, -0.06136111, -0.002650473, 0.002731125, 0.001802374, -0.001150707, 0.00009588528, -0.0000001109040, 0.0000000001264403]; +const coeffB = [1.325, 1.87, 2.5, 2.8, 2.938, 3.14, 3.37, 3.75, 4.0]; +const coeffC = [1, 1, 2, 2, 2.42, 2.63, 3, 4, 5]; + +function massKg(pressureMPa, temperatureC, volumeLiter) { + const temperatureK = temperatureC + 273.15; + let z = 1; + for (let i = 0; i < coeffA.length; i++) { + z += coeffA[i] * Math.pow(100 / temperatureK, coeffB[i]) * Math.pow(pressureMPa, coeffC[i]); + } + return pressureMPa * 1000 * 0.00201588 * volumeLiter / (8.314472 * temperatureK * z); +} + +function excelCol(index) { + let result = ""; + let value = index; + while (value > 0) { + value--; + result = String.fromCharCode(65 + (value % 26)) + result; + value = Math.floor(value / 26); + } + return result; +} + +function styleTitle(sheet, range, title) { + range.merge(); + range.values = [[title]]; + range.format = { + fill: colors.navy, + font: { bold: true, color: colors.white }, + verticalAlignment: "center", + horizontalAlignment: "left", + }; + range.format.rowHeight = 32; +} + +function styleHeader(range) { + range.format = { + fill: colors.blue, + font: { bold: true, color: colors.white }, + verticalAlignment: "center", + horizontalAlignment: "center", + wrapText: true, + borders: { preset: "all", style: "thin", color: "#B8CDD5" }, + }; + range.format.rowHeight = 30; +} + +function styleBody(range) { + range.format = { + verticalAlignment: "center", + borders: { + insideHorizontal: { style: "thin", color: "#DDE7EB" }, + bottom: { style: "thin", color: "#B8CDD5" }, + }, + }; +} + +function setWidths(sheet, widths) { + widths.forEach((width, index) => { + sheet.getRange(`${excelCol(index + 1)}:${excelCol(index + 1)}`).format.columnWidth = width; + }); +} + +const workbook = Workbook.create(); +workbook.comments.setSelf({ displayName: "User" }); +const guide = workbook.worksheets.add("验算说明"); +const params = workbook.worksheets.add("参数与系数"); +const grid = workbook.worksheets.add("120组压力质量验算"); +const raw = workbook.worksheets.add("原始报文120条"); +const detail = workbook.worksheets.add("样本计算明细"); +const summary = workbook.worksheets.add("每日与区间汇总"); + +for (const sheet of [guide, params, grid, raw, detail, summary]) sheet.showGridLines = false; + +// 参数与NIST系数 +styleTitle(params, params.getRange("A1:F1"), "氢耗算法参数与NIST实气压缩因子系数"); +params.getRange("A3:C3").values = [["参数", "取值", "说明"]]; +styleHeader(params.getRange("A3:C3")); +const parameterRows = [ + ["储氢系统总容积(L)", 520, "验算样本使用，可替换为车型确认值"], + ["动力电池容量(kWh)", 21.04, "4.5T示例；必须使用售后技术部确认值"], + ["1kg氢气等效电量(kWh/kg)", 16, "当前业务折算参数"], + ["上电稳定等待(s)", 60, "上电后60秒内样本不作为计算端点"], + ["下电提前窗口(s)", 60, "下电前60秒样本不作为计算端点"], + ["加氢压力上升阈值(MPa)", 3, "并需持续达到保持时间"], + ["加氢保持时间(s)", 300, "5分钟"], + ["纯电异常压降阈值(MPa)", 5, "30分钟内超过该值判异常"], + ["纯电异常窗口(s)", 1800, "30分钟"], + ["端点中位样本数", 5, "区间首尾各5条"], + ["最少有效样本数", 10, "不足10条不形成有效区间"], + ["样本间隔(s)", 10, "本验算原始数据间隔"], +]; +params.getRange(`A4:C${3 + parameterRows.length}`).values = parameterRows; +styleBody(params.getRange(`A4:C${3 + parameterRows.length}`)); +params.getRange(`B4:B${3 + parameterRows.length}`).format.numberFormat = "0.000"; + +params.getRange("A18:D18").values = [["系数序号", "aᵢ", "bᵢ", "cᵢ"]]; +styleHeader(params.getRange("A18:D18")); +params.getRange("A19:D27").values = coeffA.map((a, index) => [index + 1, a, coeffB[index], coeffC[index]]); +styleBody(params.getRange("A19:D27")); +params.getRange("B19:D27").format.numberFormat = "0.0000000000"; +params.getRange("A29:F33").values = [ + ["压力—质量公式", null, null, null, null, null], + ["Z = 1 + Σ[aᵢ × (100/Tₖ)^bᵢ × P^cᵢ]", null, null, null, null, null], + ["m = P × 1000 × 0.00201588 × V ÷ (8.314472 × Tₖ × Z)", null, null, null, null, null], + ["SOC平衡公式", null, null, null, null, null], + ["m_SOC = m_H₂ + C_battery × (SOC_start − SOC_end) ÷ 100 ÷ 16", null, null, null, null, null], +]; +for (const row of [29, 30, 31, 32, 33]) params.getRange(`A${row}:F${row}`).merge(); +params.getRange("A29:F29").format = { fill: colors.cyan, font: { bold: true, color: colors.navy } }; +params.getRange("A32:F32").format = { fill: colors.cyan, font: { bold: true, color: colors.navy } }; +params.getRange("A30:F31").format = { fill: colors.pale, font: { color: colors.navy }, wrapText: true }; +params.getRange("A33:F33").format = { fill: colors.pale, font: { color: colors.navy }, wrapText: true }; +setWidths(params, [28, 18, 52, 16, 16, 16]); +params.freezePanes.freezeRows(3); + +// 120组独立压力—质量验算 +const gridHeaders = ["序号", "压力P(MPa)", "温度T(℃)", "容积V(L)", "绝对温度Tₖ(K)", ...Array.from({ length: 9 }, (_, i) => `Z分项${i + 1}`), "压缩因子Z", "分子", "分母", "Excel复算质量(kg)", "系统计算质量(kg)", "绝对差值(kg)", "核验结果"]; +grid.getRange(`A1:${excelCol(gridHeaders.length)}1`).values = [gridHeaders]; +styleHeader(grid.getRange(`A1:${excelCol(gridHeaders.length)}1`)); +const pressures = [1, 3, 5, 8, 10, 12, 15, 18, 21, 25, 30, 35]; +const temperatures = [-20, -10, 0, 10, 20, 30, 40, 50, 60, 70]; +const gridInputs = []; +const gridSystem = []; +let gridIndex = 1; +for (const pressure of pressures) { + for (const temperature of temperatures) { + gridInputs.push([gridIndex++, pressure, temperature, 520]); + gridSystem.push(massKg(pressure, temperature, 520)); + } +} +grid.getRange("A2:D121").values = gridInputs; +for (let row = 2; row <= 121; row++) { + grid.getRange(`E${row}`).formulas = [[`=C${row}+273.15`]]; + for (let term = 0; term < 9; term++) { + const col = excelCol(6 + term); + const coeffRow = 19 + term; + grid.getRange(`${col}${row}`).formulas = [[`='参数与系数'!$B$${coeffRow}*POWER(100/$E${row},'参数与系数'!$C$${coeffRow})*POWER($B${row},'参数与系数'!$D$${coeffRow})`]]; + } + grid.getRange(`O${row}`).formulas = [[`=1+SUM(F${row}:N${row})`]]; + grid.getRange(`P${row}`).formulas = [[`=B${row}*1000*0.00201588*D${row}`]]; + grid.getRange(`Q${row}`).formulas = [[`=8.314472*E${row}*O${row}`]]; + grid.getRange(`R${row}`).formulas = [[`=P${row}/Q${row}`]]; + grid.getRange(`S${row}`).values = [[gridSystem[row - 2]]]; + grid.getRange(`T${row}`).formulas = [[`=ABS(R${row}-S${row})`]]; + grid.getRange(`U${row}`).formulas = [[`=IF(T${row}<=0.0000000001,"通过","不通过")`]]; +} +styleBody(grid.getRange("A2:U121")); +grid.getRange("B2:D121").format.numberFormat = "0.000"; +grid.getRange("E2:Q121").format.numberFormat = "0.0000000000"; +grid.getRange("R2:S121").format.numberFormat = "0.000000"; +grid.getRange("T2:T121").format.numberFormat = "0.000000000000"; +grid.getRange("U2:U121").conditionalFormats.add("containsText", { text: "通过", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); +grid.getRange("U2:U121").conditionalFormats.add("containsText", { text: "不通过", format: { fill: colors.red, font: { color: colors.redText, bold: true } } }); +grid.tables.add("A1:U121", true, "NISTValidation120").style = "TableStyleMedium2"; +grid.freezePanes.freezeRows(1); +grid.freezePanes.freezeColumns(5); +setWidths(grid, [8, 12, 12, 12, 15, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 16, 16, 18, 18, 18, 12]); + +// 构造120条可人工核验的原始GB/T 32960解析字段样本。 +const rawHeaders = ["序号", "报文ID", "上报时间", "数据源", "车辆状态", "充电状态", "运行模式", "燃料电池工作", "氢压(MPa)", "氢温(℃)", "储氢容积(L)", "电SOC(%)", "仪表总里程(km)", "原始解析字段JSON"]; +raw.getRange("A1:N1").values = [rawHeaders]; +styleHeader(raw.getRange("A1:N1")); +const rawRows = []; +const systemMasses = []; +const baseTime = new Date(Date.UTC(2026, 7, 26, 0, 0, 0)); +for (let index = 0; index < 120; index++) { + const eventId = `audit-32960-${String(index + 1).padStart(3, "0")}`; + const observedAt = new Date(baseTime.getTime() + index * 10_000); + let vehicleState = 1; + let chargeState = 3; + let runningMode = 2; + let fuelCellActive = 1; + let pressure; + let soc; + let mileage; + if (index < 40) { + pressure = 30 - index * 0.04; + soc = 80 - index * 0.04; + mileage = 1000 + index * 0.08; + } else if (index < 60) { + vehicleState = 2; + chargeState = 1; + runningMode = 0; + fuelCellActive = 0; + pressure = 28.4 - (index - 40) * 0.002; + soc = 78.4 + (index - 40) * (11.6 / 19); + mileage = 1003.2; + } else if (index < 80) { + chargeState = index === 60 ? 4 : 3; + runningMode = 1; + fuelCellActive = 0; + pressure = 28.36 - (index - 60) * 0.005; + soc = 90 - (index - 60) * 0.15; + mileage = 1003.2 + (index - 60) * 0.1; + } else { + pressure = 28.26 - (index - 80) * 0.06; + soc = 87 - (index - 80) * 0.04; + mileage = 1005.2 + (index - 80) * 0.12; + if (index === 119) vehicleState = 2; + } + const temperature = 30 + Math.sin(index / 10); + const mass = massKg(pressure, temperature, 520); + systemMasses.push(mass); + const rawJson = JSON.stringify({ + event_id: eventId, + vehicle_state: vehicleState, + charge_status: chargeState, + running_mode: runningMode, + fuel_cell_active: fuelCellActive, + hydrogen_max_pressure_mpa: Number(pressure.toFixed(6)), + hydrogen_max_temperature_c: Number(temperature.toFixed(6)), + soc_percent: Number(soc.toFixed(6)), + total_mileage_km: Number(mileage.toFixed(6)), + }); + rawRows.push([index + 1, eventId, observedAt, "GB32960-audit-sample", vehicleState, chargeState, runningMode, fuelCellActive, pressure, temperature, 520, soc, mileage, rawJson]); +} +raw.getRange("A2:N121").values = rawRows; +styleBody(raw.getRange("A2:N121")); +raw.getRange("C2:C121").format.numberFormat = "yyyy-mm-dd hh:mm:ss"; +raw.getRange("I2:M121").format.numberFormat = "0.000000"; +raw.getRange("N2:N121").format.wrapText = false; +raw.getRange("A123:N126").values = [ + ["人工核验说明", null, null, null, null, null, null, null, null, null, null, null, null, null], + ["本页为验算构造的原始解析字段样本，不是生产车辆真实报文。生产复核时应按相同字段结构替换为中台导出的真实32960数据。", null, null, null, null, null, null, null, null, null, null, null, null, null], + ["车辆状态：1=启动/运行，2=熄火；充电状态：1=停车充电，3=未充电，4=充电完成；运行模式：1=纯电，2=混动。", null, null, null, null, null, null, null, null, null, null, null, null, null], + ["停车充电需由车辆状态与充电状态联合判断；充电区间不计算氢耗，充电完成重新建立SOC、压力、温度和里程基线。", null, null, null, null, null, null, null, null, null, null, null, null, null], +]; +for (let row = 123; row <= 126; row++) raw.getRange(`A${row}:N${row}`).merge(); +raw.getRange("A123:N123").format = { fill: colors.cyan, font: { bold: true, color: colors.navy } }; +raw.getRange("A124:N126").format = { fill: colors.pale, font: { color: colors.grayText }, wrapText: true }; +raw.tables.add("A1:N121", true, "RawTelemetry120").style = "TableStyleMedium2"; +raw.freezePanes.freezeRows(1); +raw.freezePanes.freezeColumns(3); +setWidths(raw, [8, 24, 20, 24, 10, 10, 10, 14, 12, 12, 14, 12, 18, 90]); + +// 逐条计算明细：全部中间状态和公式均保留。 +const detailHeaders = ["序号", "报文ID", "上报时间", "压力P", "温度T", "容积V", "Tₖ", ...Array.from({ length: 9 }, (_, i) => `Z分项${i + 1}`), "Z", "分子", "分母", "Excel复算质量", "系统质量", "差值", "车辆状态", "充电状态", "运行模式", "燃料电池", "SOC", "仪表里程", "阶段", "上电60s通过", "下电60s通过", "停车充电", "状态组合有效", "参与计算", "排除原因", "区间类型", "系统区间号", "端点角色"]; +detail.getRange(`A1:${excelCol(detailHeaders.length)}1`).values = [detailHeaders]; +styleHeader(detail.getRange(`A1:${excelCol(detailHeaders.length)}1`)); +for (let row = 2; row <= 121; row++) { + const rawRow = row; + const index = row - 2; + detail.getRange(`A${row}:F${row}`).formulas = [[ + `='原始报文120条'!A${rawRow}`, + `='原始报文120条'!B${rawRow}`, + `='原始报文120条'!C${rawRow}`, + `='原始报文120条'!I${rawRow}`, + `='原始报文120条'!J${rawRow}`, + `='原始报文120条'!K${rawRow}`, + ]]; + detail.getRange(`G${row}`).formulas = [[`=E${row}+273.15`]]; + for (let term = 0; term < 9; term++) { + const col = excelCol(8 + term); + const coeffRow = 19 + term; + detail.getRange(`${col}${row}`).formulas = [[`='参数与系数'!$B$${coeffRow}*POWER(100/$G${row},'参数与系数'!$C$${coeffRow})*POWER($D${row},'参数与系数'!$D$${coeffRow})`]]; + } + detail.getRange(`Q${row}`).formulas = [[`=1+SUM(H${row}:P${row})`]]; + detail.getRange(`R${row}`).formulas = [[`=D${row}*1000*0.00201588*F${row}`]]; + detail.getRange(`S${row}`).formulas = [[`=8.314472*G${row}*Q${row}`]]; + detail.getRange(`T${row}`).formulas = [[`=R${row}/S${row}`]]; + detail.getRange(`U${row}`).values = [[systemMasses[index]]]; + detail.getRange(`V${row}`).formulas = [[`=ABS(T${row}-U${row})`]]; + detail.getRange(`W${row}:AB${row}`).formulas = [[ + `='原始报文120条'!E${rawRow}`, + `='原始报文120条'!F${rawRow}`, + `='原始报文120条'!G${rawRow}`, + `='原始报文120条'!H${rawRow}`, + `='原始报文120条'!L${rawRow}`, + `='原始报文120条'!M${rawRow}`, + ]]; + const phase = index < 40 ? "充电前运行" : index < 60 ? "停车充电" : index < 80 ? "充电后纯电" : "充电后混动"; + detail.getRange(`AC${row}`).values = [[phase]]; + detail.getRange(`AD${row}`).formulas = [[`=IF(OR(C${row}<'原始报文120条'!$C$2+TIME(0,0,'参数与系数'!$B$7),AND(C${row}>='原始报文120条'!$C$62,C${row}<'原始报文120条'!$C$62+TIME(0,0,'参数与系数'!$B$7))),"否","是")`]]; + detail.getRange(`AE${row}`).formulas = [[`=IF(OR(AND(C${row}>'原始报文120条'!$C$42-TIME(0,0,'参数与系数'!$B$8),C${row}<='原始报文120条'!$C$42),AND(C${row}>'原始报文120条'!$C$121-TIME(0,0,'参数与系数'!$B$8),C${row}<='原始报文120条'!$C$121)),"否","是")`]]; + detail.getRange(`AF${row}`).formulas = [[`=IF(AND(W${row}=2,X${row}=1),"是","否")`]]; + detail.getRange(`AG${row}`).formulas = [[`=IF(OR(AND(Y${row}=1,Z${row}=0),AND(Y${row}=2,Z${row}=1)),"是","否")`]]; + detail.getRange(`AH${row}`).formulas = [[`=IF(AND(AD${row}="是",AE${row}="是",AF${row}="否",AG${row}="是",W${row}=1),"是","否")`]]; + detail.getRange(`AI${row}`).formulas = [[`=IF(AH${row}="是","",IF(AF${row}="是","停车充电区间",IF(AD${row}="否","上电后60秒稳定窗口",IF(AE${row}="否","下电前60秒稳定窗口",IF(W${row}<>1,"车辆非运行状态","运行模式与燃料电池状态不匹配")))))`]]; + detail.getRange(`AJ${row}`).formulas = [[`=IF(AH${row}<>"是","",IF(AND(Y${row}=1,Z${row}=0),"PURE_ELECTRIC",IF(AND(Y${row}=2,Z${row}=1),"MIXED","")))`]]; + const segment = index >= 6 && index <= 34 ? 1 : index >= 66 && index <= 79 ? 2 : index >= 80 && index <= 113 ? 3 : null; + detail.getRange(`AK${row}`).values = [[segment]]; + let endpointRole = ""; + if (index >= 6 && index <= 10) endpointRole = `区间1起始候选${index - 5}`; + else if (index >= 30 && index <= 34) endpointRole = `区间1结束候选${index - 29}`; + else if (index >= 66 && index <= 70) endpointRole = `区间2起始候选${index - 65}`; + else if (index >= 75 && index <= 79) endpointRole = `区间2结束候选${index - 74}`; + else if (index >= 80 && index <= 84) endpointRole = `区间3起始候选${index - 79}`; + else if (index >= 109 && index <= 113) endpointRole = `区间3结束候选${index - 108}`; + detail.getRange(`AL${row}`).values = [[endpointRole]]; +} +styleBody(detail.getRange(`A2:AL121`)); +detail.getRange("C2:C121").format.numberFormat = "yyyy-mm-dd hh:mm:ss"; +detail.getRange("D2:V121").format.numberFormat = "0.000000"; +detail.getRange("AA2:AB121").format.numberFormat = "0.000000"; +detail.getRange("AH2:AH121").conditionalFormats.add("containsText", { text: "是", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); +detail.getRange("AH2:AH121").conditionalFormats.add("containsText", { text: "否", format: { fill: colors.gray, font: { color: colors.grayText } } }); +detail.getRange("AF2:AF121").conditionalFormats.add("containsText", { text: "是", format: { fill: colors.orange, font: { color: colors.orangeText, bold: true } } }); +detail.getRange("AL2:AL121").conditionalFormats.add("notContainsBlanks", { format: { fill: colors.cyan, font: { color: colors.navy, bold: true } } }); +detail.tables.add("A1:AL121", true, "TelemetryCalculationDetail").style = "TableStyleMedium2"; +detail.freezePanes.freezeRows(1); +detail.freezePanes.freezeColumns(3); +setWidths(detail, [8, 24, 20, 11, 11, 11, 12, ...Array(9).fill(13), 12, 14, 14, 16, 16, 16, 10, 10, 10, 12, 11, 16, 16, 13, 13, 12, 14, 12, 28, 16, 12, 18]); + +// 每日与区间汇总，公式均追溯至逐条明细。 +styleTitle(summary, summary.getRange("A1:AB1"), "120条原始报文：每日氢耗与区间计算汇总"); +summary.getRange("A3:H3").values = [["统计日期", "原始样本数", "参与计算样本", "停车充电样本", "有效区间", "混动区间", "纯电区间", "质量状态"]]; +styleHeader(summary.getRange("A3:H3")); +summary.getRange("A4").values = [["2026-08-26"]]; +summary.getRange("B4:H4").formulas = [[ + `=COUNTA('原始报文120条'!$B$2:$B$121)`, + `=COUNTIF('样本计算明细'!$AH$2:$AH$121,"是")`, + `=COUNTIF('样本计算明细'!$AF$2:$AF$121,"是")`, + `=COUNTA($A$9:$A$11)`, + `=COUNTIF($B$9:$B$11,"MIXED")`, + `=COUNTIF($B$9:$B$11,"PURE_ELECTRIC")`, + `=IF(AND(B4=120,C4>=10,F4>=1),"OK","SUSPECT")`, +]]; +styleBody(summary.getRange("A4:H4")); +summary.getRange("H4").conditionalFormats.add("containsText", { text: "OK", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); + +summary.getRange("J3:Q3").values = [["物理耗氢(kg)", "SOC差值(百分点)", "电池净放电(kWh)", "电量折氢(kg)", "SOC平衡氢耗(kg)", "混动里程(km)", "物理百公里氢耗", "SOC平衡百公里氢耗"]]; +styleHeader(summary.getRange("J3:Q3")); +summary.getRange("J4:Q4").formulas = [[ + `=SUMIF($B$9:$B$11,"MIXED",$P$9:$P$11)`, + `=SUMIF($B$9:$B$11,"MIXED",$R$9:$R$11)-SUMIF($B$9:$B$11,"MIXED",$Q$9:$Q$11)`, + `=SUMIF($B$9:$B$11,"MIXED",$T$9:$T$11)`, + `=SUMIF($B$9:$B$11,"MIXED",$U$9:$U$11)`, + `=SUMIF($B$9:$B$11,"MIXED",$V$9:$V$11)`, + `=SUMIF($B$9:$B$11,"MIXED",$Y$9:$Y$11)`, + `=IF(O4>0,J4/O4*100,"")`, + `=IF(O4>0,N4/O4*100,"")`, +]]; +styleBody(summary.getRange("J4:Q4")); +summary.getRange("J4:Q4").format.numberFormat = "0.000000"; + +summary.getRange("A7:AB7").values = [["区间", "类型", "有效样本", "起始候选行", "结束候选行", "起始报文ID", "结束报文ID", "起始时间", "结束时间", "起始压力", "结束压力", "起始温度", "结束温度", "起始质量", "结束质量", "物理耗氢", "起始SOC", "结束SOC", "SOC差值", "电池净放电", "电量折氢", "SOC平衡氢耗", "起始里程", "结束里程", "区间里程", "物理百公里", "SOC平衡百公里", "是否计入日氢耗"]]; +styleHeader(summary.getRange("A7:AB7")); +const intervalSpecs = [ + { row: 9, index: 1, type: "MIXED", sampleCount: 29, startRows: [8, 12], endRows: [32, 36] }, + { row: 10, index: 2, type: "PURE_ELECTRIC", sampleCount: 14, startRows: [68, 72], endRows: [77, 81] }, + { row: 11, index: 3, type: "MIXED", sampleCount: 34, startRows: [82, 86], endRows: [111, 115] }, +]; +for (const spec of intervalSpecs) { + const r = spec.row; + const [ss, se] = spec.startRows; + const [es, ee] = spec.endRows; + summary.getRange(`A${r}:E${r}`).values = [[spec.index, spec.type, spec.sampleCount, `${ss}:${se}`, `${es}:${ee}`]]; + summary.getRange(`F${r}`).formulas = [[`=INDEX('样本计算明细'!$B$${ss}:$B$${se},MATCH(MEDIAN('样本计算明细'!$T$${ss}:$T$${se}),'样本计算明细'!$T$${ss}:$T$${se},0))`]]; + summary.getRange(`G${r}`).formulas = [[`=INDEX('样本计算明细'!$B$${es}:$B$${ee},MATCH(MEDIAN('样本计算明细'!$T$${es}:$T$${ee}),'样本计算明细'!$T$${es}:$T$${ee},0))`]]; + summary.getRange(`H${r}`).formulas = [[`=INDEX('样本计算明细'!$C$${ss}:$C$${se},MATCH(MEDIAN('样本计算明细'!$T$${ss}:$T$${se}),'样本计算明细'!$T$${ss}:$T$${se},0))`]]; + summary.getRange(`I${r}`).formulas = [[`=INDEX('样本计算明细'!$C$${es}:$C$${ee},MATCH(MEDIAN('样本计算明细'!$T$${es}:$T$${ee}),'样本计算明细'!$T$${es}:$T$${ee},0))`]]; + const sourceCols = { J: "D", K: "D", L: "E", M: "E", N: "T", O: "T", Q: "AA", R: "AA", W: "AB", X: "AB" }; + for (const [targetCol, sourceCol] of Object.entries(sourceCols)) { + const isStart = ["J", "L", "N", "Q", "W"].includes(targetCol); + const rs = isStart ? ss : es; + const re = isStart ? se : ee; + summary.getRange(`${targetCol}${r}`).formulas = [[`=INDEX('样本计算明细'!$${sourceCol}$${rs}:$${sourceCol}$${re},MATCH(MEDIAN('样本计算明细'!$T$${rs}:$T$${re}),'样本计算明细'!$T$${rs}:$T$${re},0))`]]; + } + summary.getRange(`P${r}`).formulas = [[spec.type === "MIXED" ? `=MAX(0,N${r}-O${r})` : `=0`]]; + summary.getRange(`S${r}`).formulas = [[`=R${r}-Q${r}`]]; + summary.getRange(`T${r}`).formulas = [[spec.type === "MIXED" ? `='参数与系数'!$B$5*(Q${r}-R${r})/100` : `=""`]]; + summary.getRange(`U${r}`).formulas = [[spec.type === "MIXED" ? `=T${r}/'参数与系数'!$B$6` : `=""`]]; + summary.getRange(`V${r}`).formulas = [[spec.type === "MIXED" ? `=P${r}+U${r}` : `=""`]]; + summary.getRange(`Y${r}`).formulas = [[`=MAX(0,X${r}-W${r})`]]; + summary.getRange(`Z${r}`).formulas = [[spec.type === "MIXED" ? `=IF(Y${r}>0,P${r}/Y${r}*100,"")` : `=""`]]; + summary.getRange(`AA${r}`).formulas = [[spec.type === "MIXED" ? `=IF(Y${r}>0,V${r}/Y${r}*100,"")` : `=""`]]; + summary.getRange(`AB${r}`).values = [[spec.type === "MIXED" ? "是" : "否（仅计纯电里程）"]]; +} +styleBody(summary.getRange("A9:AB11")); +summary.getRange("H9:I11").format.numberFormat = "yyyy-mm-dd hh:mm:ss"; +summary.getRange("J9:AA11").format.numberFormat = "0.000000"; +summary.getRange("B9:B11").conditionalFormats.add("containsText", { text: "MIXED", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); +summary.getRange("B9:B11").conditionalFormats.add("containsText", { text: "PURE_ELECTRIC", format: { fill: colors.cyan, font: { color: colors.navy, bold: true } } }); + +summary.getRange("A14:AB18").values = [ + ["人工核验顺序", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], + ["1. 在“原始报文120条”核对每条车辆状态、充电状态、运行模式、压力、温度、SOC和仪表里程。", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], + ["2. 在“样本计算明细”逐项复核Tₖ、9个Z分项、压缩因子Z、分子、分母、质量及参与计算原因。", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], + ["3. 在本页核对每个区间首尾5条候选样本的中位质量、对应原始报文ID、物理耗氢、SOC修正和里程。", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], + ["4. 停车充电区间不计氢耗；充电结束后重新建立基线。纯电区间只累计纯电里程，不计入混动氢耗分母。", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], +]; +for (let row = 14; row <= 18; row++) summary.getRange(`A${row}:AB${row}`).merge(); +summary.getRange("A14:AB14").format = { fill: colors.cyan, font: { bold: true, color: colors.navy } }; +summary.getRange("A15:AB18").format = { fill: colors.pale, font: { color: colors.grayText }, wrapText: true }; +summary.freezePanes.freezeRows(7); +summary.freezePanes.freezeColumns(2); +setWidths(summary, [8, 18, 12, 14, 14, 24, 24, 20, 20, 13, 13, 13, 13, 15, 15, 15, 13, 13, 13, 16, 14, 17, 15, 15, 14, 15, 17, 20]); + +// 说明首页，关键结果均跨表引用。 +styleTitle(guide, guide.getRange("A1:H1"), "氢耗算法120组人工验算底稿"); +guide.getRange("A3:H4").merge(); +guide.getRange("A3").values = [["用途：供人工使用原始字段逐步复核压力—质量换算、充电剔除、充电后重建基线、纯电/混动分段、SOC能量修正及最终日氢耗。样本为验算构造数据，不代表生产车辆真实报文。"]]; +guide.getRange("A3:H4").format = { fill: colors.pale, font: { color: colors.navy }, wrapText: true, verticalAlignment: "center" }; +guide.getRange("A6:H6").values = [["核验项目", "数量/结果", "口径", "核验项目", "数量/结果", "口径", "状态", "备注"]]; +styleHeader(guide.getRange("A6:H6")); +guide.getRange("A7:H10").values = [ + ["NIST压力质量样本", null, "12压力×10温度", "原始报文样本", null, "含停车充电和充电后重基线", null, ""], + ["NIST通过数", null, "系统值与Excel公式差≤1e-10", "参与计算样本", null, "上/下电窗口及充电剔除后", null, ""], + ["物理耗氢", null, "仅有效混动区间", "SOC平衡氢耗", null, "物理耗氢+电池净放电折氢", null, ""], + ["混动里程", null, "仅混动区间", "纯电里程", null, "只统计，不进入混动氢耗分母", null, ""], +]; +guide.getRange("B7").formulas = [[`=COUNTA('120组压力质量验算'!$A$2:$A$121)`]]; +guide.getRange("E7").formulas = [[`=COUNTA('原始报文120条'!$A$2:$A$121)`]]; +guide.getRange("G7").formulas = [[`=IF(AND(B7=120,E7=120),"完整","不完整")`]]; +guide.getRange("B8").formulas = [[`=COUNTIF('120组压力质量验算'!$U$2:$U$121,"通过")`]]; +guide.getRange("E8").formulas = [[`='每日与区间汇总'!$C$4`]]; +guide.getRange("G8").formulas = [[`=IF(B8=120,"通过","不通过")`]]; +guide.getRange("B9").formulas = [[`='每日与区间汇总'!$J$4`]]; +guide.getRange("E9").formulas = [[`='每日与区间汇总'!$N$4`]]; +guide.getRange("G9").formulas = [[`='每日与区间汇总'!$H$4`]]; +guide.getRange("B10").formulas = [[`='每日与区间汇总'!$O$4`]]; +guide.getRange("E10").formulas = [[`=SUMIF('每日与区间汇总'!$B$9:$B$11,"PURE_ELECTRIC",'每日与区间汇总'!$Y$9:$Y$11)`]]; +guide.getRange("G10").values = [["可追溯"]]; +styleBody(guide.getRange("A7:H10")); +guide.getRange("B9:B10").format.numberFormat = "0.000000"; +guide.getRange("E9:E10").format.numberFormat = "0.000000"; +guide.getRange("G7:G10").conditionalFormats.add("containsText", { text: "通过", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); +guide.getRange("G7:G10").conditionalFormats.add("containsText", { text: "完整", format: { fill: colors.green, font: { color: colors.greenText, bold: true } } }); +guide.getRange("G7:G10").conditionalFormats.add("containsText", { text: "可追溯", format: { fill: colors.cyan, font: { color: colors.navy, bold: true } } }); + +guide.getRange("A13:H13").values = [["工作表", "核验内容", "人工操作", "", "", "", "", ""]]; +styleHeader(guide.getRange("A13:H13")); +const guideRows = [ + ["参数与系数", "所有常量和9组NIST系数", "先确认车型参数，再复核公式引用"], + ["120组压力质量验算", "120组压力/温度的完整中间值和最终差异", "逐行查看Z分项→Z→分子/分母→质量→通过结果"], + ["原始报文120条", "车辆状态、充电状态、运行模式、压力、温度、SOC、里程及JSON", "核对原始字段，不修改计算列"], + ["样本计算明细", "每条报文换算质量、边界过滤、充电识别和区间归属", "按报文ID追踪排除原因及端点候选"], + ["每日与区间汇总", "端点中位数、分段耗氢、SOC修正、里程和百公里氢耗", "从最终结果反查起止报文ID"], +]; +guide.getRange("A14:C18").values = guideRows; +for (let row = 14; row <= 18; row++) guide.getRange(`C${row}:H${row}`).merge(); +styleBody(guide.getRange("A14:H18")); +guide.getRange("A21:H21").merge(); +guide.getRange("A21").values = [["关键规则：停车充电 = 车辆状态为熄火(2) 且充电状态为停车充电(1)。充电期间不计算氢耗；充电结束后重新建立SOC、压力、温度和里程基线，仅对运行区间SOC变化进行氢气等效修正。"]]; +guide.getRange("A21:H21").format = { fill: colors.orange, font: { bold: true, color: colors.orangeText }, wrapText: true }; +guide.getRange("A21:H21").format.rowHeight = 48; +setWidths(guide, [24, 18, 34, 24, 18, 34, 14, 28]); + +await fs.mkdir(previewDir, { recursive: true }); + +const checks = {}; +for (const [name, range] of [ + ["guide", "验算说明!A1:H21"], + ["parameters", "参数与系数!A1:F33"], + ["grid", "120组压力质量验算!A1:U12"], + ["raw", "原始报文120条!A1:N15"], + ["detail", "样本计算明细!A1:AL15"], + ["summary", "每日与区间汇总!A1:AB18"], +]) { + const [sheetName, a1] = range.split("!"); + const inspection = await workbook.inspect({ kind: "table", sheetId: sheetName, range: a1, include: "values,formulas", tableMaxRows: 20, tableMaxCols: 40, maxChars: 12000 }); + checks[name] = inspection.ndjson; +} + +const formulaErrors = await workbook.inspect({ + kind: "match", + searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", + options: { useRegex: true, maxResults: 300 }, + summary: "final formula error scan", + maxChars: 10000, +}); + +for (const sheetName of ["验算说明", "参数与系数", "120组压力质量验算", "原始报文120条", "样本计算明细", "每日与区间汇总"]) { + const preview = await workbook.render({ sheetName, autoCrop: "all", scale: 0.8, format: "png" }); + await fs.writeFile(`${previewDir}/${sheetName}.png`, new Uint8Array(await preview.arrayBuffer())); +} + +const xlsx = await SpreadsheetFile.exportXlsx(workbook); +await xlsx.save(outputPath); +await fs.writeFile(`${outputDir}/verification.json`, JSON.stringify({ outputPath, checks, formulaErrors: formulaErrors.ndjson }, null, 2)); +console.log(JSON.stringify({ outputPath, previewDir, formulaErrors: formulaErrors.ndjson })); diff --git a/tmp/mileage-appeal-20260810/build_appeal.mjs b/tmp/mileage-appeal-20260810/build_appeal.mjs new file mode 100644 index 00000000..4fd93a43 --- /dev/null +++ b/tmp/mileage-appeal-20260810/build_appeal.mjs @@ -0,0 +1,446 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { SpreadsheetFile, Workbook } from "@oai/artifact-tool"; + +const evidencePath = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/mileage-appeal-20260810/evidence.json"; +const outputDir = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083"; +const outputPath = path.join(outputDir, "15辆车里程核验申诉材料_GB32960日里程及原始报文.xlsx"); +const previewDir = path.join(outputDir, "preview"); +const data = JSON.parse(await fs.readFile(evidencePath, "utf8")); + +const workbook = Workbook.create(); +const summary = workbook.worksheets.add("申诉总览"); +const notes = workbook.worksheets.add("口径与证据说明"); +const vehicleSheets = new Map(); +for (const vehicle of data.vehicles) { + const sheetName = `${String(vehicle.seq).padStart(2, "0")}-${vehicle.plate}`; + vehicleSheets.set(vehicle.vin, { sheetName, sheet: workbook.worksheets.add(sheetName) }); +} + +const palette = { + navy: "#1F4E78", + blue: "#D9EAF7", + blue2: "#EAF3F8", + yellow: "#FFF2CC", + amber: "#FCE4D6", + mileageHeader: "#C65911", + mileageLight: "#FFF2CC", + mileageStrong: "#FFD966", + green: "#E2F0D9", + gray: "#E7E6E6", + dark: "#1F2937", + muted: "#5B6573", + border: "#B7C9D6", + white: "#FFFFFF", +}; + +const thinBorders = { preset: "all", style: "thin", color: palette.border }; +const titleFormat = { + fill: palette.navy, + font: { bold: true, color: palette.white, fontSize: 16, typeface: "Microsoft YaHei" }, + horizontalAlignment: "left", + verticalAlignment: "center", +}; +const sectionFormat = { + fill: palette.blue, + font: { bold: true, color: palette.dark, fontSize: 11, typeface: "Microsoft YaHei" }, + horizontalAlignment: "left", + verticalAlignment: "center", + borders: thinBorders, +}; +const headerFormat = { + fill: palette.navy, + font: { bold: true, color: palette.white, fontSize: 9, typeface: "Microsoft YaHei" }, + horizontalAlignment: "center", + verticalAlignment: "center", + wrapText: true, + borders: thinBorders, +}; +const labelFormat = { + fill: palette.blue2, + font: { bold: true, color: palette.dark, fontSize: 9, typeface: "Microsoft YaHei" }, + horizontalAlignment: "left", + verticalAlignment: "center", + wrapText: true, + borders: thinBorders, +}; +const valueFormat = { + font: { color: palette.dark, fontSize: 9, typeface: "Microsoft YaHei" }, + verticalAlignment: "center", + wrapText: true, + borders: thinBorders, +}; + +function asDate(day) { + return new Date(`${day}T00:00:00Z`); +} + +function addDays(day, count) { + const date = asDate(day); + date.setUTCDate(date.getUTCDate() + count); + return date.toISOString().slice(0, 10); +} + +function calendarDays(start, end) { + const days = []; + for (let day = start; day <= end; day = addDays(day, 1)) days.push(day); + return days; +} + +function absDelta(a, b) { + if (a == null || b == null || Number.isNaN(Number(a)) || Number.isNaN(Number(b))) return null; + return Math.abs(Number(a) - Number(b)); +} + +function rawRange(vehicle) { + const rows = vehicle.rawEvidence.filter((row) => row.status === "OK"); + if (rows.length === 0) return "无"; + return `${rows[0].day} 至 ${rows.at(-1).day}`; +} + +function evidenceLevel(metric, raw) { + if (raw?.status === "OK") return "A-原始32960报文"; + if (metric?.source_code === "legacy_lingniu_prod") return "C-历史日统计迁移（无RAW）"; + if (metric) return "B-中台32960日统计（无RAW）"; + return "D-无中台日记录"; +} + +function rowNote(metric, raw) { + const notes = []; + if (metric?.source_code === "legacy_lingniu_prod") notes.push("历史生产库 ln_vehicle_day_mileage 迁移记录，不是原始帧"); + if (raw?.status === "OK" && !metric) notes.push("有原始报文，但中台日统计无记录"); + if (raw?.status === "OK" && metric && absDelta(metric.latest_total_mileage_km, raw.parsedTotalMileageKm) > 0.1) { + notes.push("日统计日末值与本日最后可得RAW读数不一致，详见数据源/时间；可能涉及人工源选举或跨日延迟"); + } + if (raw?.status !== "OK" && metric && metric.source_code !== "legacy_lingniu_prod") notes.push("有中台日统计，但当日未检出含仪表总里程的RAW帧"); + if (!metric && raw?.status !== "OK") notes.push("当日无中台日记录，且现行RAW历史不可用或未检出"); + return notes.join("；"); +} + +function setStandardSheet(sheet) { + sheet.showGridLines = false; + sheet.freezePanes.freezeRows(13); + sheet.freezePanes.freezeColumns(1); +} + +for (const vehicle of data.vehicles) { + const { sheetName, sheet } = vehicleSheets.get(vehicle.vin); + setStandardSheet(sheet); + sheet.getRange("A1:P1").merge(); + sheet.getRange("A1").values = [[`${vehicle.plate} 里程核验申诉明细`]]; + sheet.getRange("A1:P1").format = titleFormat; + sheet.getRange("A1:P1").format.rowHeight = 30; + + sheet.getRange("A2:P2").merge(); + sheet.getRange("A2").values = [["核验口径：GB/T 32960 车辆仪表累计总里程；原始报文列保存完整HEX，点击单元格可在公式栏复制。相邻自然日均有RAW读数时才计算RAW日差。"]]; + sheet.getRange("A2:P2").format = { fill: palette.yellow, font: { color: palette.dark, fontSize: 9, typeface: "Microsoft YaHei" }, wrapText: true, verticalAlignment: "center", borders: thinBorders }; + sheet.getRange("A2:P2").format.rowHeight = 30; + + sheet.getRange("A4:P6").format = valueFormat; + sheet.getRange("A4").values = [["车牌"]]; + sheet.getRange("B4:C4").merge(); sheet.getRange("B4").values = [[vehicle.plate]]; + sheet.getRange("D4").values = [["VIN"]]; + sheet.getRange("E4:H4").merge(); sheet.getRange("E4").values = [[vehicle.vin]]; + sheet.getRange("I4").values = [["车型"]]; + sheet.getRange("J4:K4").merge(); sheet.getRange("J4").values = [[vehicle.model]]; + sheet.getRange("L4").values = [["协议"]]; + sheet.getRange("M4:P4").merge(); sheet.getRange("M4").values = [["GB32960（仪表累计总里程）"]]; + + sheet.getRange("A5").values = [["核验区间"]]; + sheet.getRange("B5:C5").merge(); sheet.getRange("B5").values = [["2025-11-28 至 2026-07-31"]]; + sheet.getRange("D5").values = [["原TBOX合计(km)"]]; + sheet.getRange("E5:F5").merge(); sheet.getRange("E5").values = [[vehicle.sourceTboxKm]]; + sheet.getRange("G5").values = [["第三方平台合计(km)"]]; + sheet.getRange("H5:I5").merge(); sheet.getRange("H5").values = [[vehicle.platformKm]]; + sheet.getRange("J5").values = [["原表损耗(km)"]]; + sheet.getRange("K5:L5").merge(); sheet.getRange("K5").values = [[vehicle.lossKm]]; + sheet.getRange("M5").values = [["原申诉里程(km)"]]; + sheet.getRange("N5:P5").merge(); sheet.getRange("N5").values = [[vehicle.appealKm]]; + sheet.getRange("E5:F5").format.numberFormat = "#,##0.0"; + sheet.getRange("H5:I5").format.numberFormat = "#,##0.0"; + sheet.getRange("K5:L5").format.numberFormat = "#,##0.0"; + sheet.getRange("N5:P5").format.numberFormat = "#,##0.0"; + sheet.getRange("K5:L5").format.fill = palette.yellow; + + sheet.getRange("A6").values = [["异常标记"]]; + sheet.getRange("B6:D6").merge(); sheet.getRange("B6").values = [[vehicle.anomaly]]; + sheet.getRange("E6").values = [["原排查结论"]]; + sheet.getRange("F6:P6").merge(); sheet.getRange("F6").values = [[vehicle.note]]; + for (const address of ["A4", "A5", "A6", "D4", "D5", "E6", "G5", "I4", "J5", "L4", "M5"]) sheet.getRange(address).format = labelFormat; + + sheet.getRange("A8:P8").format = valueFormat; + sheet.getRange("A8").values = [["原车辆统计合计(km)"]]; + sheet.getRange("B8:C8").merge(); sheet.getRange("B8").values = [[vehicle.sourceTboxKm]]; + sheet.getRange("D8").values = [["车辆数据中台统计合计(km)"]]; + sheet.getRange("E8:F8").merge(); sheet.getRange("E8").formulas = [["=SUM(B14:B259)"]]; + sheet.getRange("G8").values = [["中台日统计有值天数"]]; + sheet.getRange("H8:I8").merge(); sheet.getRange("H8").formulas = [["=COUNT(B14:B259)"]]; + sheet.getRange("J8").values = [["RAW仪表证据天数"]]; + sheet.getRange("K8:L8").merge(); sheet.getRange("K8").formulas = [["=COUNT(F14:F259)"]]; + sheet.getRange("M8").values = [["RAW证据日期范围"]]; + sheet.getRange("N8:P8").merge(); sheet.getRange("N8").values = [[rawRange(vehicle)]]; + for (const address of ["A8", "D8", "G8", "J8", "M8"]) sheet.getRange(address).format = labelFormat; + sheet.getRange("B8:C8").format.numberFormat = "#,##0.0"; + sheet.getRange("E8:F8").format.numberFormat = "#,##0.0"; + sheet.getRange("B8:C8").format.fill = palette.blue2; + sheet.getRange("E8:F8").format.fill = palette.mileageStrong; + sheet.getRange("E8:F8").format.font = { bold: true, color: "#7F6000", fontSize: 10, typeface: "Microsoft YaHei" }; + sheet.getRange("A8:P8").format.rowHeight = 38; + + sheet.getRange("A9").values = [["证据限制"]]; + sheet.getRange("B9:P9").merge(); + sheet.getRange("B9").values = [["现行TDengine RAW历史自2026-07-01起可查询；此前仅有历史生产库日统计迁移记录。材料已按A/B/C/D分级，未将统计记录伪称为原始报文。"]]; + sheet.getRange("A9").format = labelFormat; + sheet.getRange("B9:P9").format = { ...valueFormat, fill: palette.amber }; + sheet.getRange("A9:P9").format.rowHeight = 30; + + sheet.getRange("A11:P11").merge(); + sheet.getRange("A11").values = [["每日核验明细（完整日历）"]]; + sheet.getRange("A11:P11").format = sectionFormat; + + sheet.getRange("A12:D12").merge(); + sheet.getRange("A12").values = [["车辆数据中台区间里程总计"]]; + sheet.getRange("E12:F12").merge(); + sheet.getRange("E12").formulas = [["=SUM(B14:B259)"]]; + sheet.getRange("G12:P12").merge(); + sheet.getRange("G12").values = [["计算范围：B14:B259（下方“中台当日里程(km)”列）"]]; + sheet.getRange("A12:D12").format = { + ...headerFormat, + fill: palette.mileageHeader, + horizontalAlignment: "left", + }; + sheet.getRange("E12:F12").format = { + ...valueFormat, + fill: palette.mileageStrong, + font: { bold: true, color: "#7F6000", fontSize: 11, typeface: "Microsoft YaHei" }, + numberFormat: "#,##0.0", + horizontalAlignment: "right", + }; + sheet.getRange("G12:P12").format = { + ...valueFormat, + fill: palette.mileageLight, + font: { color: palette.textMuted, fontSize: 9, typeface: "Microsoft YaHei" }, + }; + sheet.getRange("A12:P12").format.rowHeight = 30; + + const headers = ["日期", "中台当日里程(km)", "中台日末仪表总里程(km)", "日统计数据源", "日统计来源端点", "RAW解析仪表总里程(km)", "RAW相邻日差(km)", "日统计-RAW差(km)", "证据等级", "报文事件时间", "中台接收时间", "RAW来源端点", "Event ID", "Frame ID", "原始报文大小(bytes)", "原始GB32960报文HEX（完整）"]; + sheet.getRange("A13:P13").values = [headers]; + sheet.getRange("A13:P13").format = headerFormat; + sheet.getRange("A13:P13").format.rowHeight = 42; + + const metricByDay = new Map(vehicle.dailyMetrics.map((row) => [row.stat_date, row])); + const rawByDay = new Map(vehicle.rawEvidence.map((row) => [row.day, row])); + const days = calendarDays(data.period.start, data.period.end); + const rows = days.map((day) => { + const metric = metricByDay.get(day); + const raw = rawByDay.get(day); + return [ + asDate(day), + metric?.daily_mileage_km ?? null, + metric?.latest_total_mileage_km ?? null, + metric ? `${metric.platform_name || ""} / ${metric.source_code || ""}` : "", + metric?.latest_source_endpoint ?? "", + raw?.status === "OK" ? raw.parsedTotalMileageKm : null, + null, + null, + evidenceLevel(metric, raw), + raw?.status === "OK" ? raw.eventTime : "", + raw?.status === "OK" ? raw.receivedAt : "", + raw?.status === "OK" ? raw.sourceEndpoint : "", + raw?.status === "OK" ? raw.eventId : "", + raw?.status === "OK" ? raw.frameId : "", + raw?.status === "OK" ? raw.rawSizeBytes : null, + raw?.status === "OK" ? raw.rawHex : "", + ]; + }); + sheet.getRange("A14:P259").values = rows; + sheet.getRange("A14:A259").format.numberFormat = "yyyy-mm-dd"; + sheet.getRange("B14:C259").format.numberFormat = "#,##0.0"; + sheet.getRange("F14:H259").format.numberFormat = "#,##0.0"; + sheet.getRange("O14:O259").format.numberFormat = "#,##0"; + sheet.getRange("A14:P259").format = { + font: { color: palette.dark, fontSize: 8, typeface: "Microsoft YaHei" }, + verticalAlignment: "center", + borders: { insideHorizontal: { style: "thin", color: "#D9E2E8" }, bottom: { style: "thin", color: palette.border } }, + }; + sheet.getRange("A14:A259").format.horizontalAlignment = "center"; + sheet.getRange("B14:C259").format.horizontalAlignment = "right"; + sheet.getRange("F14:H259").format.horizontalAlignment = "right"; + sheet.getRange("I14:O259").format.wrapText = true; + sheet.getRange("P14:P259").format = { font: { color: palette.muted, fontSize: 7, typeface: "Consolas" }, wrapText: true, verticalAlignment: "top" }; + sheet.getRange("A14:P259").format.rowHeight = 34; + sheet.getRange("G14").values = [[null]]; + for (let row = 15; row <= 259; row += 1) { + sheet.getRange(`G${row}`).formulas = [[`=IF(OR(F${row}="",F${row - 1}=""),"",F${row}-F${row - 1})`]]; + } + for (let row = 14; row <= 259; row += 1) { + sheet.getRange(`H${row}`).formulas = [[`=IF(OR(C${row}="",F${row}=""),"",C${row}-F${row})`]]; + } + + sheet.getRange("I14:I259").conditionalFormats.add("containsText", { text: "A-", format: { fill: palette.green, font: { color: "#375623", bold: true } } }); + sheet.getRange("I14:I259").conditionalFormats.add("containsText", { text: "C-", format: { fill: palette.yellow, font: { color: "#7F6000" } } }); + sheet.getRange("I14:I259").conditionalFormats.add("containsText", { text: "D-", format: { fill: palette.gray, font: { color: palette.muted } } }); + sheet.getRange("H14:H259").conditionalFormats.addCustom("=ABS($H14)>0.1", { fill: palette.amber, font: { color: "#9C0006", bold: true } }); + + const table = sheet.tables.add("A13:P259", true, `Vehicle${String(vehicle.seq).padStart(2, "0")}Evidence`); + table.style = "TableStyleMedium2"; + table.showBandedColumns = false; + table.showFilterButton = true; + sheet.getRange("B13").format = { ...headerFormat, fill: palette.mileageHeader }; + sheet.getRange("B14:B259").format.fill = palette.mileageLight; + sheet.getRange("B14:B259").conditionalFormats.add("cellIs", { + operator: "greaterThan", + formula: 0, + format: { fill: palette.mileageStrong, font: { bold: true, color: "#7F6000" } }, + }); + + const widths = [12, 17, 20, 24, 26, 20, 17, 18, 25, 20, 20, 22, 34, 34, 14, 72]; + for (let col = 0; col < widths.length; col += 1) sheet.getRangeByIndexes(0, col, 259, 1).format.columnWidth = widths[col]; +} + +summary.showGridLines = false; +summary.freezePanes.freezeRows(4); +summary.getRange("A1:P1").merge(); +summary.getRange("A1").values = [["15辆车里程核验申诉总览"]]; +summary.getRange("A1:P1").format = titleFormat; +summary.getRange("A1:P1").format.rowHeight = 32; +summary.getRange("A2:P2").merge(); +summary.getRange("A2").values = [["结论：车辆数据中台GB32960日里程复核合计与原异常表TBOX区间里程逐车一致；第三方平台合计显著偏低。RAW原始帧按现有可查历史提供，早期记录已明确标注证据限制。"]]; +summary.getRange("A2:P2").format = { fill: palette.green, font: { bold: true, color: "#375623", fontSize: 10, typeface: "Microsoft YaHei" }, wrapText: true, verticalAlignment: "center", borders: thinBorders }; +summary.getRange("A2:P2").format.rowHeight = 34; + +summary.getRange("A3:B3").merge(); summary.getRange("A3").values = [["全部15辆车·原车辆统计(km)"]]; +summary.getRange("C3").formulas = [["=F20"]]; +summary.getRange("D3:E3").merge(); summary.getRange("D3").values = [["车辆数据中台统计(km)"]]; +summary.getRange("F3").formulas = [["=G20"]]; +summary.getRange("G3:H3").merge(); summary.getRange("G3").values = [["第三方平台统计(km)"]]; +summary.getRange("I3").formulas = [["=H20"]]; +summary.getRange("J3:K3").merge(); summary.getRange("J3").values = [["平台漏计差异(km)"]]; +summary.getRange("L3").formulas = [["=I20"]]; +summary.getRange("M3:N3").merge(); summary.getRange("M3").values = [["RAW仪表证据(天)"]]; +summary.getRange("O3:P3").merge(); summary.getRange("O3").formulas = [["=M20"]]; +for (const address of ["A3:B3", "D3:E3", "G3:H3", "J3:K3", "M3:N3"]) summary.getRange(address).format = labelFormat; +for (const address of ["C3", "F3", "I3", "L3", "O3:P3"]) summary.getRange(address).format = { ...valueFormat, fill: palette.mileageLight, font: { bold: true, color: "#7F6000", fontSize: 10, typeface: "Microsoft YaHei" }, horizontalAlignment: "right" }; +summary.getRange("C3:L3").format.numberFormat = "#,##0.0"; +summary.getRange("O3:P3").format.numberFormat = "#,##0"; +summary.getRange("A3:P3").format.rowHeight = 34; + +const summaryHeaders = ["序号", "车型", "车牌", "VIN", "核验区间", "原TBOX合计(km)", "中台复核合计(km)", "第三方平台合计(km)", "损耗(km)", "损耗率", "申诉里程(km)", "日统计天数", "RAW证据天数", "RAW证据范围", "核验结论", "原排查结论"]; +summary.getRange("A4:P4").values = [summaryHeaders]; +summary.getRange("A4:P4").format = headerFormat; +summary.getRange("A4:P4").format.rowHeight = 40; +const summaryRows = data.vehicles.map((vehicle) => [ + vehicle.seq, vehicle.model, vehicle.plate, vehicle.vin, "2025-11-28 至 2026-07-31", vehicle.sourceTboxKm, null, vehicle.platformKm, null, null, vehicle.appealKm, null, null, rawRange(vehicle), null, vehicle.note, +]); +summary.getRange("A5:P19").values = summaryRows; +for (let index = 0; index < data.vehicles.length; index += 1) { + const row = 5 + index; + const { sheetName } = vehicleSheets.get(data.vehicles[index].vin); + summary.getRange(`G${row}`).formulas = [[`='${sheetName}'!E8`]]; + summary.getRange(`I${row}`).formulas = [[`=F${row}-H${row}`]]; + summary.getRange(`J${row}`).formulas = [[`=IF(F${row}=0,"",I${row}/F${row})`]]; + summary.getRange(`L${row}`).formulas = [[`='${sheetName}'!H8`]]; + summary.getRange(`M${row}`).formulas = [[`='${sheetName}'!K8`]]; + summary.getRange(`O${row}`).formulas = [[`=IF(ABS(G${row}-F${row})<0.1,"中台复核合计与原TBOX一致","需复核中台合计")`]]; +} +summary.getRange("A20:E20").merge(); summary.getRange("A20").values = [["合计"]]; +summary.getRange("F20").formulas = [["=SUM(F5:F19)"]]; +summary.getRange("G20").formulas = [["=SUM(G5:G19)"]]; +summary.getRange("H20").formulas = [["=SUM(H5:H19)"]]; +summary.getRange("I20").formulas = [["=SUM(I5:I19)"]]; +summary.getRange("J20").formulas = [["=IF(F20=0,\"\",I20/F20)"]]; +summary.getRange("K20").formulas = [["=SUM(K5:K19)"]]; +summary.getRange("L20").formulas = [["=SUM(L5:L19)"]]; +summary.getRange("M20").formulas = [["=SUM(M5:M19)"]]; +summary.getRange("N20:P20").merge(); summary.getRange("N20").values = [["15车合计；RAW证据为现行历史可查询范围"]]; +summary.getRange("A20:P20").format = { fill: palette.blue, font: { bold: true, color: palette.dark, fontSize: 9, typeface: "Microsoft YaHei" }, borders: thinBorders, verticalAlignment: "center", wrapText: true }; +summary.getRange("A5:P19").format = { font: { color: palette.dark, fontSize: 9, typeface: "Microsoft YaHei" }, verticalAlignment: "center", wrapText: true, borders: { insideHorizontal: { style: "thin", color: "#D9E2E8" }, bottom: { style: "thin", color: palette.border } } }; +summary.getRange("F5:I20").format.numberFormat = "#,##0.0"; +summary.getRange("J5:J20").format.numberFormat = "0.00%"; +summary.getRange("K5:K20").format.numberFormat = "#,##0.0"; +summary.getRange("L5:M20").format.numberFormat = "#,##0"; +summary.getRange("I5:I19").format.fill = palette.yellow; +summary.getRange("O5:O19").conditionalFormats.add("containsText", { text: "一致", format: { fill: palette.green, font: { color: "#375623", bold: true } } }); +const summaryTable = summary.tables.add("A4:P19", true, "AppealSummary"); +summaryTable.style = "TableStyleMedium2"; +summaryTable.showFilterButton = true; +const summaryWidths = [8, 9, 14, 22, 22, 16, 18, 18, 15, 12, 16, 13, 14, 24, 26, 54]; +for (let col = 0; col < summaryWidths.length; col += 1) summary.getRangeByIndexes(0, col, 20, 1).format.columnWidth = summaryWidths[col]; +summary.getRange("A5:P19").format.rowHeight = 42; + +notes.showGridLines = false; +notes.getRange("A1:H1").merge(); +notes.getRange("A1").values = [["核验口径与证据说明"]]; +notes.getRange("A1:H1").format = titleFormat; +notes.getRange("A1:H1").format.rowHeight = 32; +notes.getRange("A3:H3").merge(); notes.getRange("A3").values = [["1. 核验口径"]]; notes.getRange("A3:H3").format = sectionFormat; +notes.getRange("A4:B8").values = [ + ["协议", "GB/T 32960"], + ["仪表总里程字段", "gb32960.vehicle.total_mileage_km"], + ["仪表总里程含义", "车辆仪表盘累计总里程，单位 km；不同协议累计值不可拼接。"], + ["中台当日里程", "生产 vehicle_daily_mileage 的 GB32960 日统计值；本材料逐车求和后与原异常表TBOX合计核对。"], + ["RAW相邻日差", "仅相邻两个自然日都存在RAW仪表读数时计算；缺任一日则留空，不把跨日缺口跳变塞进单日。"], +]; +notes.getRange("A4:A8").format = labelFormat; +notes.getRange("B4:H8").merge(true); +notes.getRange("B4:H8").format = valueFormat; +notes.getRange("A10:H10").merge(); notes.getRange("A10").values = [["2. 证据分级"]]; notes.getRange("A10:H10").format = sectionFormat; +notes.getRange("A11:B14").values = [ + ["A-原始32960报文", "中台TDengine保存的完整GB32960十六进制报文，且解析结果明确包含仪表总里程字段。"], + ["B-中台32960日统计", "中台日统计存在，但当日未检出含仪表总里程的RAW帧。"], + ["C-历史日统计迁移", "来自历史生产库 lingniu_prod.ln_vehicle_day_mileage 的迁移记录，不是原始帧。"], + ["D-无中台日记录", "当日中台日统计无记录，且现行RAW历史不可用或未检出。"], +]; +notes.getRange("A11:A14").format = labelFormat; +notes.getRange("B11:H14").merge(true); +notes.getRange("B11:H14").format = valueFormat; +notes.getRange("A11:H11").format.fill = palette.green; +notes.getRange("A13:H13").format.fill = palette.yellow; +notes.getRange("A14:H14").format.fill = palette.gray; + +notes.getRange("A16:H16").merge(); notes.getRange("A16").values = [["3. 数据来源与限制"]]; notes.getRange("A16:H16").format = sectionFormat; +notes.getRange("A17:B21").values = [ + ["原异常表", "/Users/lingniu/Documents/里程异常统计表_20251128-20260731-回复.xlsx"], + ["中台日统计API", "http://115.29.187.205:20200/api/stats/daily-metrics"], + ["中台RAW历史API", "http://115.29.187.205:20200/api/history/raw-frames"], + ["RAW可查范围", "本次提取范围2026-07-01至2026-07-31；15车共取得334个含仪表总里程字段的完整原始帧。"], + ["重要限制", "2026-07-01之前现行TDengine没有可查询RAW历史；只能提供历史生产库日统计迁移记录。对外申诉时应如实说明，并向原TSP/车厂补调更早期原始报文归档。"], +]; +notes.getRange("A17:A21").format = labelFormat; +notes.getRange("B17:H21").merge(true); +notes.getRange("B17:H21").format = valueFormat; +notes.getRange("A21:H21").format.fill = palette.amber; + +notes.getRange("A23:H23").merge(); notes.getRange("A23").values = [["4. 使用建议"]]; notes.getRange("A23:H23").format = sectionFormat; +notes.getRange("A24:H27").values = [ + ["① 先提交“申诉总览”，证明15车中台GB32960日里程合计与原TBOX合计一致。", null, null, null, null, null, null, null], + ["② 每车附对应Sheet，重点引用A类RAW证据、事件时间、接收时间和完整HEX。", null, null, null, null, null, null, null], + ["③ 对第三方缺失日，用完整日历中的D级空缺、前后仪表累计变化和原排查结论说明漏传。", null, null, null, null, null, null, null], + ["④ 申诉若要求全区间逐日原始报文，需由原TSP/车厂补调2026-07-01之前归档；本材料未虚构缺失证据。", null, null, null, null, null, null, null], +]; +for (let row = 24; row <= 27; row += 1) notes.getRange(`A${row}:H${row}`).merge(); +notes.getRange("A24:H27").format = { ...valueFormat, fill: palette.blue2 }; +notes.getRange("A4:H27").format.rowHeight = 30; +notes.getRange("A1:A27").format.columnWidth = 22; +for (let col = 1; col < 8; col += 1) notes.getRangeByIndexes(0, col, 27, 1).format.columnWidth = 18; + +await fs.mkdir(outputDir, { recursive: true }); +await fs.mkdir(previewDir, { recursive: true }); + +const inspection = {}; +inspection.summary = (await workbook.inspect({ kind: "table", range: "申诉总览!A1:P20", include: "values,formulas", tableMaxRows: 24, tableMaxCols: 16, maxChars: 22000 })).ndjson; +inspection.formulaErrors = (await workbook.inspect({ kind: "match", searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", options: { useRegex: true, maxResults: 300 }, summary: "final formula error scan" })).ndjson; +await fs.writeFile(path.join(outputDir, "verification.ndjson"), `${inspection.summary}\n${inspection.formulaErrors}\n`, "utf8"); + +const previewSheets = ["申诉总览", "口径与证据说明", ...data.vehicles.map((vehicle) => vehicleSheets.get(vehicle.vin).sheetName)]; +for (let index = 0; index < previewSheets.length; index += 1) { + const sheetName = previewSheets[index]; + const range = sheetName === "申诉总览" ? "A1:P20" : sheetName === "口径与证据说明" ? "A1:H27" : "A1:P24"; + const preview = await workbook.render({ sheetName, range, scale: 1, format: "png" }); + const safeName = sheetName.replaceAll(/[\\/:*?\[\]]/g, "_"); + await fs.writeFile(path.join(previewDir, `${String(index + 1).padStart(2, "0")}-${safeName}.png`), new Uint8Array(await preview.arrayBuffer())); +} + +const output = await SpreadsheetFile.exportXlsx(workbook); +await output.save(outputPath); +console.log(JSON.stringify({ outputPath, sheets: previewSheets.length, previewDir }, null, 2)); diff --git a/tmp/mileage-appeal-20260810/collect_evidence.mjs b/tmp/mileage-appeal-20260810/collect_evidence.mjs new file mode 100644 index 00000000..25a90ce3 --- /dev/null +++ b/tmp/mileage-appeal-20260810/collect_evidence.mjs @@ -0,0 +1,158 @@ +import fs from "node:fs/promises"; + +const baseUrl = "http://115.29.187.205:20200"; +const outputPath = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/mileage-appeal-20260810/evidence.json"; +const vehicles = [ + { seq: 1, model: "18T", plate: "粤A05995F", vin: "LNXNEGRR7SR321370", sourceTboxKm: 26180.6, platformKm: 4208.9, lossKm: 21971.7, lossRatePct: 83.92, anomaly: "里程损失，tbox异常", appealKm: 41497.1, note: "区间内仪表盘开始里程是576.6,结束里程是41497.1,差值40920.5,小于申诉里程;2026-05-11到2026-07-28间的数据没上传,仪表盘里程差了36306.6,2025-12-25到2026-01-01间还存在里程跳变导致里程缺失" }, + { seq: 2, model: "18T", plate: "粤A05839F", vin: "LNXNEGRR8SR321376", sourceTboxKm: 41030.3, platformKm: 19763.6, lossKm: 21266.7, lossRatePct: 51.83, anomaly: "里程损失", appealKm: 41030.3, note: "存在很多数据缺失导致的里程跳变,统计里程缺失,特别是7月份仪表里程从7月1号的16341.2到7月31号的41848.9,相差25507.7,计算的总里程只有8470" }, + { seq: 3, model: "18T", plate: "粤A09369F", vin: "LNXNEGRR6SR321473", sourceTboxKm: 40381.5, platformKm: 24994.0, lossKm: 15387.5, lossRatePct: 38.11, anomaly: "里程损失", appealKm: 40381.5, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 4, model: "18T", plate: "粤A06658F", vin: "LNXNEGRR5SR321397", sourceTboxKm: 52386.5, platformKm: 33064.5, lossKm: 19322.0, lossRatePct: 36.88, anomaly: "里程损失", appealKm: 52386.5, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 5, model: "18T", plate: "粤A05700F", vin: "LNXNEGRR8SR321393", sourceTboxKm: 60054.3, platformKm: 41352.9, lossKm: 18701.4, lossRatePct: 31.14, anomaly: "里程损失", appealKm: 60054.3, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 6, model: "18T", plate: "粤A05906F", vin: "LNXNEGRR2SR319462", sourceTboxKm: 37216.8, platformKm: 26711.4, lossKm: 10505.4, lossRatePct: 28.23, anomaly: "里程损失", appealKm: 37216.8, note: "最新数据只到2026-06-30,仪表盘最后里程为28638.5,没有7月份的数据;也存在很多数据缺失导致的里程缺失,例如2026-03-26到2026-04-01,2026-04-21到2026-05-01间没上传数据" }, + { seq: 7, model: "4.5T", plate: "粤AGP9346", vin: "LB9A32A28R0LS1722", sourceTboxKm: 32337.2, platformKm: 28685.4, lossKm: 3651.8, lossRatePct: 11.29, anomaly: "里程损失", appealKm: 32337.2, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 8, model: "4.5T", plate: "粤AGP2017", vin: "LB9A32A28R0LS1445", sourceTboxKm: 48164.9, platformKm: 44357.1, lossKm: 3807.8, lossRatePct: 7.91, anomaly: "里程损失", appealKm: 48164.9, note: "存在很多数据缺失导致的里程跳变,统计里程缺失,而且区间内最后的仪表里程是2026-07-31的47337.5,小于申诉里程" }, + { seq: 9, model: "4.5T", plate: "粤AGP5165", vin: "LB9A32A25R0LS1385", sourceTboxKm: 32906.4, platformKm: 30302.3, lossKm: 2604.1, lossRatePct: 7.91, anomaly: "里程损失", appealKm: 32906.4, note: "存在很多数据缺失导致的里程跳变,统计里程缺失,而且区间内最后的仪表里程是2026-07-30的31994.6,2026-07-31没有数据上传" }, + { seq: 10, model: "4.5T", plate: "粤AGP4355", vin: "LB9A32A26R0LS1606", sourceTboxKm: 55151.0, platformKm: 51525.8, lossKm: 3625.2, lossRatePct: 6.57, anomaly: "里程损失", appealKm: 55151.0, note: "区间内仪表盘开始里程是1898.6,结束里程是55229.6,差值53331,小于申诉里程,也存在很多数据缺失导致的里程缺失" }, + { seq: 11, model: "4.5T", plate: "粤AGP5646", vin: "LB9A32A22R0LS1702", sourceTboxKm: 39935.0, platformKm: 37319.8, lossKm: 2615.2, lossRatePct: 6.55, anomaly: "里程损失", appealKm: 39935.0, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 12, model: "4.5T", plate: "粤AGP3692", vin: "LB9A32A27R0LS1727", sourceTboxKm: 64195.0, platformKm: 60242.6, lossKm: 3952.4, lossRatePct: 6.16, anomaly: "里程损失", appealKm: 64195.0, note: "存在很多数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 13, model: "4.5T", plate: "粤AGP9782", vin: "LB9A32A28R0LS1705", sourceTboxKm: 50216.4, platformKm: 47123.8, lossKm: 3092.6, lossRatePct: 6.16, anomaly: "里程损失", appealKm: 50216.4, note: "存在很多数据缺失导致的里程跳变,统计里程缺失,如:2026-05-21数据缺失导致里程少了1300多" }, + { seq: 14, model: "4.5T", plate: "粤AGP3027", vin: "LB9A32A22R0LS1599", sourceTboxKm: 31180.9, platformKm: 29260.1, lossKm: 1920.8, lossRatePct: 6.16, anomaly: "里程损失", appealKm: 31180.9, note: "区间内仪表盘开始里程是933.5,结束里程是31245.3,差值30311.8,小于申诉里程,也存在一些数据缺失导致的里程跳变,统计里程缺失" }, + { seq: 15, model: "4.5T", plate: "粤AGP9751", vin: "LB9A32A28R0LS1574", sourceTboxKm: 17944.6, platformKm: 16861.0, lossKm: 1083.6, lossRatePct: 6.04, anomaly: "里程损失", appealKm: 17944.6, note: "区间内仪表盘开始里程是205.2,结束里程是17796.8,差值17591.6,小于申诉里程,也存在一些数据缺失导致的里程缺失" }, +]; + +function dateString(date) { + return date.toISOString().slice(0, 10); +} + +function addDays(day, count) { + const date = new Date(`${day}T00:00:00Z`); + date.setUTCDate(date.getUTCDate() + count); + return dateString(date); +} + +function dateRange(start, end) { + const out = []; + for (let day = start; day <= end; day = addDays(day, 1)) out.push(day); + return out; +} + +async function fetchJson(url, attempts = 4) { + let lastError; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + try { + const response = await fetch(url, { signal: AbortSignal.timeout(30000) }); + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); + return await response.json(); + } catch (error) { + lastError = error; + if (attempt < attempts) await new Promise((resolve) => setTimeout(resolve, 250 * attempt)); + } + } + throw lastError; +} + +async function mapConcurrent(items, concurrency, worker) { + const results = new Array(items.length); + let cursor = 0; + async function run() { + while (true) { + const index = cursor; + cursor += 1; + if (index >= items.length) return; + results[index] = await worker(items[index], index); + } + } + await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, run)); + return results; +} + +async function fetchDaily(vehicle) { + const params = new URLSearchParams({ + vin: vehicle.vin, + protocol: "GB32960", + dateFrom: "2025-11-28", + dateTo: "2026-07-31", + limit: "500", + includeTotal: "true", + }); + const data = await fetchJson(`${baseUrl}/api/stats/daily-metrics?${params}`); + return data.items ?? []; +} + +async function fetchRawEvidence({ vehicle, day }) { + const params = new URLSearchParams({ + protocol: "GB32960", + vin: vehicle.vin, + dateFrom: `${day} 00:00:00`, + dateTo: `${addDays(day, 1)} 00:00:00`, + orderBy: "eventTime", + limit: "20", + includeFields: "true", + includePayload: "true", + fields: "gb32960.vehicle.total_mileage_km", + }); + let data = await fetchJson(`${baseUrl}/api/history/raw-frames?${params}`); + let evidence = (data.items ?? []).find((item) => item?.parsed_fields?.["gb32960.vehicle.total_mileage_km"] != null && String(item.event_time ?? "").startsWith(day)); + if (!evidence && (data.items ?? []).length === 20) { + params.set("limit", "100"); + data = await fetchJson(`${baseUrl}/api/history/raw-frames?${params}`); + evidence = (data.items ?? []).find((item) => item?.parsed_fields?.["gb32960.vehicle.total_mileage_km"] != null && String(item.event_time ?? "").startsWith(day)); + } + if (!evidence) return { day, status: "NO_RAW_ODOMETER_FRAME" }; + return { + day, + status: "OK", + ts: evidence.ts ?? "", + eventTime: evidence.event_time ?? "", + receivedAt: evidence.received_at ?? "", + frameId: evidence.frame_id ?? "", + eventId: evidence.event_id ?? "", + messageIdHex: evidence.message_id_hex ?? "", + rawSizeBytes: evidence.raw_size_bytes ?? null, + rawHex: evidence.raw_hex ?? "", + parseStatus: evidence.parse_status ?? "", + parseError: evidence.parse_error ?? "", + sourceEndpoint: evidence.source_endpoint ?? "", + parsedTotalMileageKm: Number(evidence.parsed_fields["gb32960.vehicle.total_mileage_km"]), + }; +} + +const dailyLists = await mapConcurrent(vehicles, 6, fetchDaily); +const rawDays = dateRange("2026-07-01", "2026-07-31"); +const rawTasks = vehicles.flatMap((vehicle) => rawDays.map((day) => ({ vehicle, day }))); +let completed = 0; +const rawResults = await mapConcurrent(rawTasks, 10, async (task) => { + const result = await fetchRawEvidence(task); + completed += 1; + if (completed % 50 === 0 || completed === rawTasks.length) console.log(`raw evidence ${completed}/${rawTasks.length}`); + return { vin: task.vehicle.vin, ...result }; +}); + +const byVinRaw = new Map(); +for (const item of rawResults) { + if (!byVinRaw.has(item.vin)) byVinRaw.set(item.vin, []); + byVinRaw.get(item.vin).push(item); +} + +const result = { + generatedAt: new Date().toISOString(), + baseUrl, + period: { start: "2025-11-28", end: "2026-07-31" }, + protocol: "GB32960", + parsedMileageField: "gb32960.vehicle.total_mileage_km", + rawEvidenceAvailability: { start: "2026-07-01", end: "2026-07-31", note: "现行TDengine RAW历史自2026-07-01起；此前中台仅保留历史生产库日统计迁移记录，不能伪称原始帧。" }, + vehicles: vehicles.map((vehicle, index) => ({ + ...vehicle, + dailyMetrics: dailyLists[index], + rawEvidence: (byVinRaw.get(vehicle.vin) ?? []).sort((a, b) => a.day.localeCompare(b.day)), + })), +}; + +await fs.writeFile(outputPath, JSON.stringify(result, null, 2), "utf8"); +console.log(JSON.stringify({ + outputPath, + vehicles: result.vehicles.length, + dailyRows: result.vehicles.reduce((sum, item) => sum + item.dailyMetrics.length, 0), + rawEvidenceRows: result.vehicles.reduce((sum, item) => sum + item.rawEvidence.filter((row) => row.status === "OK").length, 0), + rawMissingDays: result.vehicles.reduce((sum, item) => sum + item.rawEvidence.filter((row) => row.status !== "OK").length, 0), +}, null, 2)); diff --git a/tmp/mileage-appeal-20260810/inspect_source.mjs b/tmp/mileage-appeal-20260810/inspect_source.mjs new file mode 100644 index 00000000..79b78c0e --- /dev/null +++ b/tmp/mileage-appeal-20260810/inspect_source.mjs @@ -0,0 +1,51 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const inputPath = "/Users/lingniu/Documents/里程异常统计表_20251128-20260731-回复.xlsx"; +const outputDir = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/mileage-appeal-20260810/source-preview"; +await fs.mkdir(outputDir, { recursive: true }); + +const input = await FileBlob.load(inputPath); +const workbook = await SpreadsheetFile.importXlsx(input); + +const overview = await workbook.inspect({ + kind: "workbook,sheet,table,drawing", + maxChars: 20000, + tableMaxRows: 12, + tableMaxCols: 20, + tableMaxCellChars: 180, +}); +await fs.writeFile(path.join(outputDir, "overview.ndjson"), overview.ndjson, "utf8"); + +const sheets = []; +for (let i = 0; ; i += 1) { + let sheet; + try { + sheet = workbook.worksheets.getItemAt(i); + } catch { + break; + } + if (!sheet) break; + const name = sheet.name; + const used = sheet.getUsedRange(); + const usedAddress = used?.address ?? null; + const info = await workbook.inspect({ + kind: "region,computedStyle,formula", + sheetId: name, + range: usedAddress ?? "A1:Z60", + maxChars: 30000, + tableMaxRows: 80, + tableMaxCols: 30, + tableMaxCellChars: 240, + options: { maxResults: 200 }, + }); + await fs.writeFile(path.join(outputDir, `${String(i + 1).padStart(2, "0")}-${name.replaceAll(/[\\/:*?\[\]]/g, "_")}.ndjson`), info.ndjson, "utf8"); + const preview = await workbook.render({ sheetName: name, autoCrop: "all", scale: 1, format: "png" }); + const previewPath = path.join(outputDir, `${String(i + 1).padStart(2, "0")}-${name.replaceAll(/[\\/:*?\[\]]/g, "_")}.png`); + await fs.writeFile(previewPath, new Uint8Array(await preview.arrayBuffer())); + sheets.push({ index: i, name, usedAddress, previewPath }); +} + +await fs.writeFile(path.join(outputDir, "sheets.json"), JSON.stringify(sheets, null, 2), "utf8"); +console.log(JSON.stringify(sheets, null, 2)); diff --git a/tmp/mileage-appeal-20260810/make_contact_sheet.py b/tmp/mileage-appeal-20260810/make_contact_sheet.py new file mode 100644 index 00000000..ef6a5864 --- /dev/null +++ b/tmp/mileage-appeal-20260810/make_contact_sheet.py @@ -0,0 +1,28 @@ +import sys +from pathlib import Path +from PIL import Image, ImageDraw, ImageFont + +source = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083/preview") +output = source / "contact-sheet.jpg" +files = sorted(source.glob("*.png")) +thumb_w, thumb_h = 620, 360 +label_h = 34 +cols = 2 +rows = (len(files) + cols - 1) // cols +canvas = Image.new("RGB", (cols * thumb_w, rows * (thumb_h + label_h)), "white") +draw = ImageDraw.Draw(canvas) +font = ImageFont.load_default() + +for index, file in enumerate(files): + image = Image.open(file).convert("RGB") + image.thumbnail((thumb_w - 12, thumb_h - 12)) + x0 = (index % cols) * thumb_w + y0 = (index // cols) * (thumb_h + label_h) + x = x0 + (thumb_w - image.width) // 2 + y = y0 + 6 + canvas.paste(image, (x, y)) + draw.rectangle((x0, y0, x0 + thumb_w - 1, y0 + thumb_h + label_h - 1), outline="#B7C9D6", width=1) + draw.text((x0 + 8, y0 + thumb_h + 8), file.stem, fill="#1F2937", font=font) + +canvas.save(output, quality=88) +print(output) diff --git a/tmp/mileage-appeal-20260810/verify_final.mjs b/tmp/mileage-appeal-20260810/verify_final.mjs new file mode 100644 index 00000000..8cb85410 --- /dev/null +++ b/tmp/mileage-appeal-20260810/verify_final.mjs @@ -0,0 +1,27 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool"; + +const workbookPath = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083/15辆车里程核验申诉材料_GB32960日里程及原始报文.xlsx"; +const outputDir = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/019fe9b9-eb60-7d22-8c0a-c70851a48083/final-qa"; +await fs.mkdir(outputDir, { recursive: true }); +const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(workbookPath)); + +const sheets = await workbook.inspect({ kind: "sheet", include: "id,name", maxChars: 8000 }); +const summary = await workbook.inspect({ kind: "table", range: "申诉总览!A1:P20", include: "values,formulas", tableMaxRows: 22, tableMaxCols: 16, maxChars: 18000 }); +const vehicleTop = await workbook.inspect({ kind: "table", range: "02-粤A05839F!A4:P14", include: "values,formulas", tableMaxRows: 14, tableMaxCols: 16, maxChars: 12000 }); +const errors = await workbook.inspect({ kind: "match", searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", options: { useRegex: true, maxResults: 300 }, summary: "reimported final formula error scan" }); +await fs.writeFile(path.join(outputDir, "final-inspect.ndjson"), `${sheets.ndjson}\n${summary.ndjson}\n${vehicleTop.ndjson}\n${errors.ndjson}\n`, "utf8"); + +const vehicleSheetNames = [ + "01-粤A05995F", "02-粤A05839F", "03-粤A09369F", "04-粤A06658F", "05-粤A05700F", + "06-粤A05906F", "07-粤AGP9346", "08-粤AGP2017", "09-粤AGP5165", "10-粤AGP4355", + "11-粤AGP5646", "12-粤AGP3692", "13-粤AGP9782", "14-粤AGP3027", "15-粤AGP9751", +]; +for (let index = 0; index < vehicleSheetNames.length; index += 1) { + const sheetName = vehicleSheetNames[index]; + const preview = await workbook.render({ sheetName, range: "A232:P259", scale: 1, format: "png" }); + await fs.writeFile(path.join(outputDir, `${String(index + 1).padStart(2, "0")}-${sheetName}.png`), new Uint8Array(await preview.arrayBuffer())); +} + +console.log(JSON.stringify({ workbookPath, outputDir, sheetCount: 17, formulaErrorScan: errors.ndjson }, null, 2)); diff --git a/tmp/seg-mileage-import-20260907/db.py b/tmp/seg-mileage-import-20260907/db.py new file mode 100644 index 00000000..d36dc561 --- /dev/null +++ b/tmp/seg-mileage-import-20260907/db.py @@ -0,0 +1,17 @@ +import pymysql, pathlib, re, json +def connect(): + env={} + for name in ['base.env','stat-writer.env']: + for line in pathlib.Path('/opt/lingniu-go-native/env/'+name).read_text().splitlines(): + if '=' in line and not line.startswith('#'): + k,v=line.split('=',1);env[k]=v.strip().strip('\"').strip("'") + m=re.fullmatch(r'([^:]+):(.*?)@tcp\(([^:]+):(\d+)\)/([^?]+).*',env['MYSQL_DSN']) + assert m, 'Unexpected DSN format' + return pymysql.connect(user=m[1],password=m[2],host=m[3],port=int(m[4]),database=m[5],charset='utf8mb4',cursorclass=pymysql.cursors.DictCursor,autocommit=False) +if __name__=='__main__': + c=connect() + with c.cursor() as q: + for table in ['vehicle_daily_mileage_source','vehicle_daily_mileage','vehicle_data_source','vehicle_identifier','vehicle']: + q.execute('SHOW COLUMNS FROM '+table); print(table,json.dumps(q.fetchall(),default=str,ensure_ascii=False)) + q.execute("SELECT * FROM vehicle_data_source WHERE platform_name LIKE '%赛格%' OR source_code LIKE '%seg%' OR source_ip LIKE '%seg%'"); print('SEG_SOURCES',json.dumps(q.fetchall(),default=str,ensure_ascii=False)) + c.close() diff --git a/tmp/seg-mileage-import-20260907/import.py b/tmp/seg-mileage-import-20260907/import.py new file mode 100644 index 00000000..16b114e1 --- /dev/null +++ b/tmp/seg-mileage-import-20260907/import.py @@ -0,0 +1,80 @@ +from db import connect +import json, pathlib, collections, sys, decimal +root=pathlib.Path(__file__).parent +rows=json.loads((root/'rows.json').read_text()) +c=connect(); q=c.cursor() +def query(sql,args=()): + q.execute(sql,args);return q.fetchall() +maps=collections.defaultdict(set) +for r in query("SELECT vin,plate FROM vehicle UNION SELECT vin,plate FROM vehicle_identifier WHERE enabled=1 UNION SELECT vin,plate FROM vehicle_identity_binding"): + if r['plate'] and r['vin']:maps[r['plate'].strip()].add(r['vin']) +unmapped=[]; mapped=[] +for r in rows: + vins=maps[r['plate']] + if len(vins)!=1:unmapped.append(dict(r,matching_vins=sorted(vins)));continue + mapped.append(dict(r,vin=next(iter(vins)))) +assert len({(r['vin'],r['date']) for r in mapped})==len(mapped),'VIN/date collision' +source_ids=[r['id'] for r in query("SELECT id FROM vehicle_data_source WHERE protocol='JT808' AND (source_code='saige' OR platform_name LIKE '%%赛格%%')")] +existing=query("SELECT s.* FROM vehicle_daily_mileage_source s LEFT JOIN vehicle_data_source d ON d.protocol=s.protocol AND d.source_ip=s.source_ip WHERE s.protocol='JT808' AND s.stat_date BETWEEN '2026-07-01' AND '2026-08-31' AND (d.source_code='saige' OR d.platform_name LIKE '%%赛格%%' OR s.platform_name LIKE '%%赛格%%' OR s.source_key LIKE '%%saige%%')") +finals=query("SELECT * FROM vehicle_daily_mileage WHERE protocol='JT808' AND stat_date BETWEEN '2026-07-01' AND '2026-08-31'") +keys={(r['vin'],str(r['stat_date'])) for r in existing} +keys.update((r['vin'],str(r['stat_date'])) for r in finals if r['source_id'] in source_ids) +pending=[r for r in mapped if (r['vin'],r['date']) not in keys] +summary=dict(input_rows=len(rows),mapped_rows=len(mapped),unmapped_rows=len(unmapped),unmapped_plates=sorted({r['plate'] for r in unmapped}),existing_saige_skipped=len(mapped)-len(pending),insert_candidates=len(pending),insert_vehicles=len({r['vin'] for r in pending}),insert_total_km=str(sum((decimal.Decimal(r['km']) for r in pending),decimal.Decimal(0)))) +(root/'preflight.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)) +(root/'unmapped.json').write_text(json.dumps(unmapped,ensure_ascii=False,indent=2)) +(root/'pending.json').write_text(json.dumps(pending,ensure_ascii=False,indent=2)) +print(json.dumps(summary,ensure_ascii=False)) +if '--apply' not in sys.argv: + c.rollback(); c.close(); sys.exit(0) +# Historical dates only. Lock every affected vehicle/day source range and result +# in deterministic order before rechecking existence. No UPDATE statements. +c.rollback() +pending.sort(key=lambda r:(r['vin'],r['date'])) +before=[]; locked=[]; skipped_race=0 +for r in pending: + args=(r['vin'],r['date']) + ss=query("SELECT * FROM vehicle_daily_mileage_source WHERE vin=%s AND stat_date=%s AND protocol='JT808' FOR UPDATE",args) + ff=query("SELECT * FROM vehicle_daily_mileage WHERE vin=%s AND stat_date=%s AND protocol='JT808' FOR UPDATE",args) + before.append(dict(vin=r['vin'],date=r['date'],sources=ss,final=ff)) + has=any(s['source_ip'] in ('222.66.200.68','58.33.87.196','manual-saige-excel') or 'saige' in s['source_key'].lower() or '赛格' in (s['platform_name'] or '') for s in ss) or any(f['source_id'] in source_ids for f in ff) + if has:skipped_race+=1;continue + locked.append((r,ff)) +(root/'backup-before.json').write_text(json.dumps(before,default=str,ensure_ascii=False)) +ds=query("SELECT * FROM vehicle_data_source WHERE protocol='JT808' AND source_ip='manual-saige-excel'") +if ds: + assert ds[0]['source_code']=='saige' and ds[0]['platform_name']=='赛格' + sid=ds[0]['id'] +else: + q.execute("INSERT INTO vehicle_data_source (protocol,source_ip,platform_name,source_code,source_kind,trust_priority,enabled,remark) VALUES ('JT808','manual-saige-excel','赛格','saige','PLATFORM',100,1,%s)",('赛格Excel每日里程导入；仅补缺，不覆盖已有记录；报表未提供累计总里程',)) + sid=q.lastrowid +inserted=[]; final_inserted=0 +for r,ff in locked: + key='JT808:'+r['phone']+'@PLATFORM:saige_excel' + reason='manual_saige_excel_import_20260907;file='+r['file']+';row='+str(r['row']) + q.execute("""INSERT INTO vehicle_daily_mileage_source + (vin,stat_date,protocol,source_key,source_ip,source_endpoint,phone,platform_name,daily_mileage_km,sample_count,quality_status,quality_reason,is_selected) + VALUES (%s,%s,'JT808',%s,'manual-saige-excel','manual-excel-import',%s,'赛格',%s,1,'OK',%s,%s)""",(r['vin'],r['date'],key,r['phone'],r['km'],reason,int(not ff))) + assert q.rowcount==1 + if not ff: + q.execute("INSERT INTO vehicle_daily_mileage (vin,stat_date,protocol,source_id,daily_mileage_km) VALUES (%s,%s,'JT808',%s,%s)",(r['vin'],r['date'],sid,r['km'])) + assert q.rowcount==1;final_inserted+=1 + inserted.append(dict(r,source_key=key)) +# Verify every old row is byte-for-byte equivalent as typed query results. +for b in before: + args=(b['vin'],b['date']) + now=query("SELECT * FROM vehicle_daily_mileage_source WHERE vin=%s AND stat_date=%s AND protocol='JT808'",args) + lookup={s['source_key']:s for s in now} + assert all(lookup[s['source_key']]==s for s in b['sources']), 'Existing source changed' + if b['final']: + assert query("SELECT * FROM vehicle_daily_mileage WHERE vin=%s AND stat_date=%s AND protocol='JT808'",args)==b['final'],'Existing final changed' +for r in inserted: + v=query("SELECT daily_mileage_km,latest_total_mileage_km,first_total_mileage_km FROM vehicle_daily_mileage_source WHERE vin=%s AND stat_date=%s AND protocol='JT808' AND source_key=%s",(r['vin'],r['date'],r['source_key'])) + assert len(v)==1 and v[0]['daily_mileage_km']==decimal.Decimal(r['km']) and v[0]['latest_total_mileage_km'] is None and v[0]['first_total_mileage_km'] is None +result=dict(summary,inserted_sources=len(inserted),inserted_final_rows=final_inserted,preserved_existing_final_rows=len(locked)-final_inserted,concurrent_skipped=skipped_race,source_id=sid,existing_rows_verified_unchanged=True) +(root/'inserted.json').write_text(json.dumps(inserted,ensure_ascii=False)) +c.commit() +result['committed']=True +(root/'result.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)) +print(json.dumps(result,ensure_ascii=False)) +c.close() diff --git a/tmp/seg-mileage-import-20260907/prepare.py b/tmp/seg-mileage-import-20260907/prepare.py new file mode 100644 index 00000000..9f2bc130 --- /dev/null +++ b/tmp/seg-mileage-import-20260907/prepare.py @@ -0,0 +1,25 @@ +import xlrd, json, pathlib, decimal, hashlib, re +out=pathlib.Path(__file__).parent +rows=[]; summary=[]; seen={} +for p in sorted(pathlib.Path('/Users/lingniu/Documents').glob('里程统计报表*.xls')): + s=xlrd.open_workbook(str(p)).sheet_by_index(0) + meta={'file':p.name,'sha256':hashlib.sha256(p.read_bytes()).hexdigest(),'rows':s.nrows-1} + if s.cell_value(0,2)!='日期': + meta.update(excluded='No daily date column',nonzero=sum(decimal.Decimal(str(s.cell_value(i,3)))!=0 for i in range(1,s.nrows))) + summary.append(meta); continue + dates=[] + for i in range(1,s.nrows): + plate,phone,day,km=map(lambda x:str(x).strip(),s.row_values(i)) + import datetime + datetime.date.fromisoformat(day) + assert re.fullmatch(r'(?:\d+|\d{1,3}(?:,\d{3})+)(?:\.\d+)?',km), km + amount=decimal.Decimal(km.replace(',','')); assert amount.is_finite() and amount>=0 + key=(plate,day) + assert key not in seen, ('duplicate',key) + seen[key]=amount; dates.append(day) + rows.append(dict(plate=plate,phone=phone,date=day,km=str(amount),file=p.name,row=i+1)) + meta.update(date_min=min(dates),date_max=max(dates),vehicles=len({r['plate'] for r in rows if r['file']==p.name})) + summary.append(meta) +(out/'rows.json').write_text(json.dumps(rows,ensure_ascii=False)) +(out/'input-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)) +print(json.dumps(summary,ensure_ascii=False,indent=2)) diff --git a/tmp/seg-mileage-import-20260907/verify.py b/tmp/seg-mileage-import-20260907/verify.py new file mode 100644 index 00000000..b2b58e11 --- /dev/null +++ b/tmp/seg-mileage-import-20260907/verify.py @@ -0,0 +1,15 @@ +from db import connect +import pathlib,json,decimal,collections +p=pathlib.Path(__file__).parent +inserted=json.loads((p/'inserted.json').read_text()) +c=connect();q=c.cursor() +q.execute("SELECT vin,stat_date,source_key,daily_mileage_km,latest_total_mileage_km FROM vehicle_daily_mileage_source WHERE source_ip='manual-saige-excel' AND stat_date BETWEEN '2026-07-01' AND '2026-08-31'") +actual={(r['vin'],str(r['stat_date']),r['source_key']):r for r in q.fetchall()} +months=collections.defaultdict(lambda:dict(rows=0,total_km=decimal.Decimal(0))) +for r in inserted: + a=actual[(r['vin'],r['date'],r['source_key'])] + assert a['daily_mileage_km']==decimal.Decimal(r['km']) and a['latest_total_mileage_km'] is None + m=months[r['date'][:7]];m['rows']+=1;m['total_km']+=a['daily_mileage_km'] +v=dict(verified_rows=len(inserted),months=dict(months),verified_total_km=sum((m['total_km'] for m in months.values()),decimal.Decimal(0))) +(p/'verification.json').write_text(json.dumps(v,default=str,ensure_ascii=False,indent=2)) +print(json.dumps(v,default=str,ensure_ascii=False));c.close() diff --git a/tmp/vehicle-mileage-selection-019fa1ac-db39-7933-bb0e-30dd63cc25bd/build.mjs b/tmp/vehicle-mileage-selection-019fa1ac-db39-7933-bb0e-30dd63cc25bd/build.mjs new file mode 100644 index 00000000..453db348 --- /dev/null +++ b/tmp/vehicle-mileage-selection-019fa1ac-db39-7933-bb0e-30dd63cc25bd/build.mjs @@ -0,0 +1,638 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { SpreadsheetFile, Workbook } from "@oai/artifact-tool"; + +const workspaceRoot = "/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest"; +const workDir = path.join( + workspaceRoot, + "tmp/vehicle-mileage-selection-019fa1ac-db39-7933-bb0e-30dd63cc25bd", +); +const outputDir = path.join( + workspaceRoot, + "outputs/019fa1ac-db39-7933-bb0e-30dd63cc25bd", +); +const outputPath = path.join( + outputDir, + "江浙沪四类车型近三个月GPS里程清单_20260427-20260726.xlsx", +); +const masterPath = path.join( + workspaceRoot, + "outputs/jt808-provider-audit-20260717/oneos-vehicle-master.json", +); + +const dateFrom = "2026-04-27"; +const dateTo = "2026-07-26"; +const periodDays = 91; +const queryAsOf = "2026-07-28"; +const mileageProtocol = "JT808"; + +function assert(condition, message) { + if (!condition) throw new Error(message); +} + +function postProductionAPI(endpoint, body) { + const remoteCommand = + `. /opt/lingniu-vehicle-platform/env/access-tokens.env; ` + + `curl -fsS -H "Authorization: Bearer $ADMIN_TOKEN" ` + + `-H "Content-Type: application/json" --data-binary @- ` + + `http://127.0.0.1:20300${endpoint}`; + const result = spawnSync( + "ssh", + ["-o", "BatchMode=yes", "root@115.29.187.205", remoteCommand], + { + input: JSON.stringify(body), + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }, + ); + if (result.status !== 0) { + throw new Error( + `Production API failed for ${endpoint}: ${result.stderr || result.stdout}`, + ); + } + const payload = JSON.parse(result.stdout); + if (!payload.data) { + throw new Error(`Production API returned no data for ${endpoint}`); + } + return payload.data; +} + +function excelDate(isoDate) { + return new Date(`${isoDate}T00:00:00+08:00`); +} + +function datesBetween(start, end) { + const result = []; + const cursor = new Date(`${start}T00:00:00+08:00`); + const last = new Date(`${end}T00:00:00+08:00`); + while (cursor <= last) { + result.push(cursor.toLocaleDateString("en-CA", { timeZone: "Asia/Shanghai" })); + cursor.setDate(cursor.getDate() + 1); + } + return result; +} + +function residentLocation(rawCity) { + const parts = String(rawCity || "").split("-"); + const province = parts[0] || ""; + const city = parts.slice(1).join("-") || province; + return { province, city }; +} + +function vehicleCategory(model) { + const normalized = String(model || ""); + if (normalized.includes("4.5吨") && !normalized.includes("冷链")) { + return "4.5T普货"; + } + if (normalized.includes("冷链")) return "冷链"; + if (normalized.includes("18吨")) return "18T"; + if (normalized.includes("49吨")) return "49T"; + return ""; +} + +function isJiangZheHu(city) { + return ["江苏省-", "浙江省-", "上海市-"].some((prefix) => + String(city || "").startsWith(prefix), + ); +} + +async function loadMasterVehicles() { + const source = JSON.parse(await fs.readFile(masterPath, "utf8")); + const rows = source.sheets?.[0]?.rows || []; + assert(rows.length > 1, "OneOS vehicle master is empty"); + const headers = rows[0].values; + const headerIndex = Object.fromEntries(headers.map((value, index) => [value, index])); + const requiredHeaders = [ + "车牌号", + "VIN", + "运营城市", + "品牌", + "型号", + "运营状态", + "车辆状态", + "客户名称", + ]; + for (const header of requiredHeaders) { + assert(headerIndex[header] !== undefined, `Missing master header: ${header}`); + } + const result = []; + for (const row of rows.slice(1)) { + const values = [...row.values, ...Array(headers.length).fill("")]; + const cityRaw = values[headerIndex["运营城市"]]; + const model = values[headerIndex["型号"]]; + const category = vehicleCategory(model); + if (!category || !isJiangZheHu(cityRaw)) continue; + const vin = String(values[headerIndex["VIN"]] || "").trim(); + const plate = String(values[headerIndex["车牌号"]] || "").trim(); + if (!vin || !plate) continue; + const location = residentLocation(cityRaw); + result.push({ + category, + plate, + vin, + brand: String(values[headerIndex["品牌"]] || "").trim(), + model: String(model || "").trim(), + province: location.province, + city: location.city, + cityRaw: String(cityRaw || "").trim(), + operationStatus: String(values[headerIndex["运营状态"]] || "").trim(), + vehicleStatus: String(values[headerIndex["车辆状态"]] || "").trim(), + customerName: ["租赁", "自营"].includes( + String(values[headerIndex["运营状态"]] || "").trim(), + ) + ? String(values[headerIndex["客户名称"]] || "").trim() + : "", + }); + } + return result; +} + +function queryRanking(vins) { + if (vins.length === 0) return { ranking: [], asOf: "" }; + return postProductionAPI("/api/v2/statistics/mileage", { + dateFrom, + dateTo, + vins, + protocol: mileageProtocol, + }); +} + +function rankingMap(data) { + return new Map((data.ranking || []).map((row) => [row.vin, row])); +} + +function selectCategoryVehicles(category, candidates) { + const categoryCandidates = candidates.filter((row) => row.category === category); + assert( + categoryCandidates.length >= 10, + `${category} has only ${categoryCandidates.length} Jiang-Zhe-Hu candidates`, + ); + const masterByVIN = new Map(categoryCandidates.map((row) => [row.vin, row])); + const suzhouVINs = categoryCandidates + .filter((row) => row.city === "苏州市") + .map((row) => row.vin); + const otherVINs = categoryCandidates + .filter((row) => row.city !== "苏州市") + .map((row) => row.vin); + const suzhouRanking = queryRanking(suzhouVINs); + const otherRanking = queryRanking(otherVINs); + const combined = [ + ...(suzhouRanking.ranking || []).map((row) => ({ + ...row, + suzhouPreferred: true, + })), + ...(otherRanking.ranking || []).map((row) => ({ + ...row, + suzhouPreferred: false, + })), + ]; + assert(combined.length >= 10, `${category} has fewer than 10 vehicles with mileage`); + return combined.slice(0, 10).map((ranked, index) => ({ + ...masterByVIN.get(ranked.vin), + rank: index + 1, + sourcePeriodMileageKm: Number(ranked.mileageKm || 0), + sourceActiveDays: Number(ranked.activeDays || 0), + suzhouPreferred: ranked.suzhouPreferred, + statisticsAsOf: suzhouRanking.asOf || otherRanking.asOf || "", + })); +} + +function queryDailyMileage(vins) { + const items = []; + let offset = 0; + const limit = 1000; + let total = Number.POSITIVE_INFINITY; + while (offset < total) { + const page = postProductionAPI("/api/mileage/daily", { + dateFrom, + dateTo, + vins, + protocol: mileageProtocol, + deduplicate: true, + limit, + offset, + }); + const pageItems = page.items || []; + items.push(...pageItems); + total = Number(page.total || 0); + offset += pageItems.length; + if (pageItems.length === 0) break; + } + return items; +} + +function styleTitle(sheet, rangeAddress, title) { + const range = sheet.getRange(rangeAddress); + range.merge(); + range.values = [[title]]; + range.format = { + fill: "#0F4C5C", + font: { bold: true, color: "#FFFFFF", size: 16 }, + verticalAlignment: "center", + horizontalAlignment: "left", + }; + range.format.rowHeight = 34; +} + +function styleHeader(range) { + range.format = { + fill: "#DCEEF2", + font: { bold: true, color: "#12343B" }, + verticalAlignment: "center", + horizontalAlignment: "center", + wrapText: true, + borders: { + bottom: { style: "medium", color: "#78A7B2" }, + }, + }; + range.format.rowHeight = 30; +} + +function styleNote(range) { + range.format = { + fill: "#F3F8FA", + font: { color: "#385A64", size: 10 }, + wrapText: true, + verticalAlignment: "center", + }; +} + +function addCategoryConditionalFormatting(range) { + range.conditionalFormats.addCustom('=$A5="4.5T普货"', { + fill: "#FFF4CC", + font: { bold: true, color: "#7A5300" }, + }); + range.conditionalFormats.addCustom('=$A5="冷链"', { + fill: "#DDF3FF", + font: { bold: true, color: "#075985" }, + }); + range.conditionalFormats.addCustom('=$A5="18T"', { + fill: "#E8E1FF", + font: { bold: true, color: "#5B21B6" }, + }); + range.conditionalFormats.addCustom('=$A5="49T"', { + fill: "#DFF5E6", + font: { bold: true, color: "#166534" }, + }); +} + +const masterVehicles = await loadMasterVehicles(); +const categories = ["4.5T普货", "冷链", "18T", "49T"]; +const selectedVehicles = categories.flatMap((category) => + selectCategoryVehicles(category, masterVehicles), +); +assert(selectedVehicles.length === 40, "Expected exactly 40 selected vehicles"); +assert( + new Set(selectedVehicles.map((row) => row.vin)).size === 40, + "Selected VINs are not unique", +); + +const dailySourceRows = queryDailyMileage(selectedVehicles.map((row) => row.vin)); +const dailyByVehicleDate = new Map(); +for (const row of dailySourceRows) { + assert( + row.source === mileageProtocol, + `Unexpected mileage source for ${row.vin} on ${row.date}: ${row.source}`, + ); + const key = `${row.vin}|${row.date}`; + assert(!dailyByVehicleDate.has(key), `Duplicate deduplicated daily row: ${key}`); + dailyByVehicleDate.set(key, row); +} + +const dates = datesBetween(dateFrom, dateTo); +assert(dates.length === periodDays, `Expected ${periodDays} dates, got ${dates.length}`); + +const dailyRows = []; +for (const vehicle of selectedVehicles) { + for (const date of dates) { + const source = dailyByVehicleDate.get(`${vehicle.vin}|${date}`); + dailyRows.push({ + date, + ...vehicle, + dailyMileageKm: source ? Number(source.dailyMileageKm || 0) : null, + source: source?.source || "", + recordStatus: source ? "有记录" : "缺失", + }); + } +} +assert(dailyRows.length === 40 * periodDays, "Daily row count mismatch"); + +for (const vehicle of selectedVehicles) { + const computed = dailyRows + .filter((row) => row.vin === vehicle.vin) + .reduce((sum, row) => sum + (row.dailyMileageKm ?? 0), 0); + const difference = Math.abs(computed - vehicle.sourcePeriodMileageKm); + assert( + difference < 0.02, + `${vehicle.plate} total mismatch: daily=${computed}, stats=${vehicle.sourcePeriodMileageKm}`, + ); +} + +const workbook = Workbook.create(); +const notesSheet = workbook.worksheets.add("口径说明"); +const summarySheet = workbook.worksheets.add("车辆汇总"); +const dailySheet = workbook.worksheets.add("每日里程"); + +for (const sheet of [notesSheet, summarySheet, dailySheet]) { + sheet.showGridLines = false; +} + +// 口径说明 +styleTitle(notesSheet, "A1:H1", "江浙沪四类车型近三个月 GPS 里程清单"); +notesSheet.getRange("A3:B11").values = [ + ["项目", "口径"], + ["统计区间", `${dateFrom} 至 ${dateTo}（${periodDays} 个自然日，完整日）`], + ["区域范围", "常驻市（OneOS 运营城市）属于江苏、浙江、上海"], + ["选车规则", "每类先按苏州区间总里程降序选取，不足 10 辆时按江浙沪其他城市区间总里程降序补齐"], + ["车型定义", "4.5T普货=型号含4.5吨且不含冷链；冷链=型号含冷链；18T=型号含18吨；49T=型号含49吨"], + ["里程口径", "仅使用 JT808 定位终端/GPS 侧累计里程计算，不使用 GB32960、宇通 MQTT 等仪表/车端累计里程"], + ["缺失处理", "逐日表保留全部车辆×日期组合；无生产记录的日期留空，并标记为“缺失”，不擅自按0计算"], + ["主数据快照", "OneOS车辆信息快照：2026-07-17；常驻市取“运营城市”字段"], + ["里程数据截至", `${queryAsOf} 查询，统计截止至 ${dateTo}；来源协议固定为 ${mileageProtocol}`], +]; +styleHeader(notesSheet.getRange("A3:B3")); +styleNote(notesSheet.getRange("A4:B11")); +notesSheet.getRange("A4:A11").format.font = { bold: true, color: "#12343B" }; +notesSheet.getRange("A3:B11").format.borders = { + outside: { style: "thin", color: "#B9D3DA" }, + insideHorizontal: { style: "thin", color: "#DCE8EC" }, +}; +notesSheet.getRange("A13:E18").values = [ + ["分类", "车辆数", "苏州优先数", "GPS区间总里程(km)", "GPS日均里程(km/天)"], + ["4.5T普货", null, null, null, null], + ["冷链", null, null, null, null], + ["18T", null, null, null, null], + ["49T", null, null, null, null], + ["合计", null, null, null, null], +]; +styleHeader(notesSheet.getRange("A13:E13")); +notesSheet.getRange("B14").formulas = [["=COUNTIF('车辆汇总'!$A$5:$A$44,A14)"]]; +notesSheet.getRange("B14:B17").fillDown(); +notesSheet.getRange("C14").formulas = [ + ["=COUNTIFS('车辆汇总'!$A$5:$A$44,A14,'车辆汇总'!$I$5:$I$44,\"苏州市\")"], +]; +notesSheet.getRange("C14:C17").fillDown(); +notesSheet.getRange("D14").formulas = [ + ["=SUMIF('车辆汇总'!$A$5:$A$44,A14,'车辆汇总'!$O$5:$O$44)"], +]; +notesSheet.getRange("D14:D17").fillDown(); +notesSheet.getRange("E14").formulas = [["=D14/(B14*'车辆汇总'!$N$5)"]]; +notesSheet.getRange("E14:E17").fillDown(); +notesSheet.getRange("B18").formulas = [["=SUM(B14:B17)"]]; +notesSheet.getRange("C18").formulas = [["=SUM(C14:C17)"]]; +notesSheet.getRange("D18").formulas = [["=SUM(D14:D17)"]]; +notesSheet.getRange("E18").formulas = [["=D18/(B18*'车辆汇总'!$N$5)"]]; +notesSheet.getRange("D14:E18").format.numberFormat = "#,##0.0"; +notesSheet.getRange("A18:E18").format = { + fill: "#E8F1F4", + font: { bold: true, color: "#12343B" }, + borders: { top: { style: "double", color: "#78A7B2" } }, +}; +notesSheet.getRange("A13:E18").format.borders = { + outside: { style: "thin", color: "#B9D3DA" }, + insideHorizontal: { style: "thin", color: "#DCE8EC" }, +}; +notesSheet.getRange("A:A").format.columnWidth = 18; +notesSheet.getRange("B:B").format.columnWidth = 64; +notesSheet.getRange("C:E").format.columnWidth = 18; +notesSheet.getRange("A4:B11").format.rowHeight = 34; +notesSheet.freezePanes.freezeRows(3); + +// 车辆汇总 +styleTitle(summarySheet, "A1:Q1", "四类车型各10辆｜近三个月 GPS 里程汇总"); +summarySheet.getRange("A2:Q2").merge(); +summarySheet.getRange("A2").values = [[ + `范围：江浙沪常驻车辆；苏州优先；统计区间 ${dateFrom} 至 ${dateTo}。仅使用 JT808 GPS 里程，区间总里程由“每日里程”明细公式汇总。`, +]]; +styleNote(summarySheet.getRange("A2:Q2")); +const summaryHeaders = [ + "分类", + "类内排名", + "车牌", + "VIN", + "品牌", + "型号", + "常驻省", + "常驻城市原值", + "常驻市", + "苏州优先", + "运营状态", + "车辆状态", + "客户名称", + "区间天数", + "GPS区间总里程(km)", + "GPS日均里程(km/天)", + "有记录天数", +]; +summarySheet.getRange("A4:Q4").values = [summaryHeaders]; +styleHeader(summarySheet.getRange("A4:Q4")); +const summaryValues = selectedVehicles.map((vehicle) => [ + vehicle.category, + vehicle.rank, + vehicle.plate, + vehicle.vin, + vehicle.brand, + vehicle.model, + vehicle.province, + vehicle.cityRaw, + vehicle.city, + vehicle.suzhouPreferred ? "是" : "否", + vehicle.operationStatus, + vehicle.vehicleStatus, + vehicle.customerName, + periodDays, + null, + null, + null, +]); +summarySheet.getRange(`A5:Q${4 + summaryValues.length}`).values = summaryValues; +const dailyEndRow = 4 + dailyRows.length; +summarySheet.getRange("O5").formulas = [[ + `=SUMIFS('每日里程'!$J$5:$J$${dailyEndRow},'每日里程'!$E$5:$E$${dailyEndRow},D5)`, +]]; +summarySheet.getRange("O5:O44").fillDown(); +summarySheet.getRange("P5").formulas = [["=O5/N5"]]; +summarySheet.getRange("P5:P44").fillDown(); +summarySheet.getRange("Q5").formulas = [[ + `=COUNTIFS('每日里程'!$E$5:$E$${dailyEndRow},D5,'每日里程'!$L$5:$L$${dailyEndRow},"有记录")`, +]]; +summarySheet.getRange("Q5:Q44").fillDown(); +summarySheet.getRange("B5:B44").format.numberFormat = "0"; +summarySheet.getRange("N5:N44").format.numberFormat = "0"; +summarySheet.getRange("O5:P44").format.numberFormat = "#,##0.0"; +summarySheet.getRange("Q5:Q44").format.numberFormat = "0"; +summarySheet.getRange("A5:Q44").format.verticalAlignment = "center"; +summarySheet.getRange("A5:Q44").format.borders = { + insideHorizontal: { style: "thin", color: "#E2ECEF" }, +}; +addCategoryConditionalFormatting(summarySheet.getRange("A5:A44")); +summarySheet.getRange("O5:O44").conditionalFormats.add("dataBar", { + color: "#2A9D8F", + gradient: true, +}); +summarySheet.getRange("J5:J44").conditionalFormats.add("containsText", { + text: "是", + format: { + fill: "#E5F7ED", + font: { bold: true, color: "#166534" }, + }, +}); +const summaryTable = summarySheet.tables.add("A4:Q44", true, "VehicleMileageSummary"); +summaryTable.style = "TableStyleMedium2"; +summaryTable.showFilterButton = true; +summarySheet.freezePanes.freezeRows(4); +summarySheet.freezePanes.freezeColumns(4); +const summaryWidths = [ + 14, 10, 14, 22, 12, 30, 12, 20, 12, 10, 12, 12, 30, 10, 18, 18, 14, +]; +summaryWidths.forEach((width, index) => { + summarySheet.getRangeByIndexes(0, index, 44, 1).format.columnWidth = width; +}); +summarySheet.getRange("A5:Q44").format.rowHeight = 23; + +// 每日里程 +styleTitle(dailySheet, "A1:L1", "40辆车逐日 GPS 里程明细"); +dailySheet.getRange("A2:L2").merge(); +dailySheet.getRange("A2").values = [[ + `共 ${selectedVehicles.length} 辆 × ${periodDays} 天 = ${dailyRows.length.toLocaleString("zh-CN")} 行；仅统计 JT808 GPS 里程，缺失日期留空并单独标记。`, +]]; +styleNote(dailySheet.getRange("A2:L2")); +const dailyHeaders = [ + "日期", + "分类", + "类内排名", + "车牌", + "VIN", + "品牌", + "型号", + "常驻市", + "客户名称", + "GPS每日里程(km)", + "来源", + "记录状态", +]; +dailySheet.getRange("A4:L4").values = [dailyHeaders]; +styleHeader(dailySheet.getRange("A4:L4")); +const dailyValues = dailyRows.map((row) => [ + excelDate(row.date), + row.category, + row.rank, + row.plate, + row.vin, + row.brand, + row.model, + row.city, + row.customerName, + row.dailyMileageKm, + row.source, + row.recordStatus, +]); +dailySheet.getRange(`A5:L${dailyEndRow}`).values = dailyValues; +dailySheet.getRange(`A5:A${dailyEndRow}`).format.numberFormat = "yyyy-mm-dd"; +dailySheet.getRange(`C5:C${dailyEndRow}`).format.numberFormat = "0"; +dailySheet.getRange(`J5:J${dailyEndRow}`).format.numberFormat = "#,##0.0"; +dailySheet.getRange(`A5:L${dailyEndRow}`).format.borders = { + insideHorizontal: { style: "thin", color: "#EDF2F4" }, +}; +dailySheet.getRange(`L5:L${dailyEndRow}`).conditionalFormats.add("containsText", { + text: "缺失", + format: { + fill: "#FFF0F0", + font: { bold: true, color: "#B42318" }, + }, +}); +dailySheet.getRange(`J5:J${dailyEndRow}`).conditionalFormats.add("dataBar", { + color: "#5CA4A9", + gradient: true, +}); +const dailyTable = dailySheet.tables.add( + `A4:L${dailyEndRow}`, + true, + "VehicleDailyMileage", +); +dailyTable.style = "TableStyleMedium2"; +dailyTable.showFilterButton = true; +dailySheet.freezePanes.freezeRows(4); +dailySheet.freezePanes.freezeColumns(5); +const dailyWidths = [14, 14, 10, 14, 22, 12, 30, 12, 30, 18, 16, 12]; +dailyWidths.forEach((width, index) => { + dailySheet.getRangeByIndexes(0, index, dailyEndRow, 1).format.columnWidth = width; +}); +dailySheet.getRange(`A5:L${dailyEndRow}`).format.rowHeight = 21; + +// Compact verification before export. +const summaryInspect = await workbook.inspect({ + kind: "table", + range: "车辆汇总!A1:Q16", + include: "values,formulas", + tableMaxRows: 16, + tableMaxCols: 17, + maxChars: 12000, +}); +await fs.writeFile(path.join(workDir, "summary-inspect.ndjson"), summaryInspect.ndjson); + +const dailyInspect = await workbook.inspect({ + kind: "table", + range: "每日里程!A1:L20", + include: "values,formulas", + tableMaxRows: 20, + tableMaxCols: 12, + maxChars: 12000, +}); +await fs.writeFile(path.join(workDir, "daily-inspect.ndjson"), dailyInspect.ndjson); + +const formulaErrors = await workbook.inspect({ + kind: "match", + searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A", + options: { useRegex: true, maxResults: 300 }, + summary: "final formula error scan", + maxChars: 12000, +}); +await fs.writeFile(path.join(workDir, "formula-errors.ndjson"), formulaErrors.ndjson); + +for (const [sheetName, range, fileName] of [ + ["口径说明", "A1:H19", "preview-notes.png"], + ["车辆汇总", "A1:Q18", "preview-summary.png"], + ["每日里程", "A1:L26", "preview-daily.png"], +]) { + const preview = await workbook.render({ + sheetName, + range, + scale: 1.25, + format: "png", + }); + await fs.writeFile( + path.join(workDir, fileName), + new Uint8Array(await preview.arrayBuffer()), + ); +} + +await fs.mkdir(outputDir, { recursive: true }); +const output = await SpreadsheetFile.exportXlsx(workbook); +await output.save(outputPath); + +console.log( + JSON.stringify( + { + outputPath, + selected: selectedVehicles.map((vehicle) => ({ + category: vehicle.category, + rank: vehicle.rank, + plate: vehicle.plate, + vin: vehicle.vin, + brand: vehicle.brand, + model: vehicle.model, + residentCity: vehicle.city, + suzhouPreferred: vehicle.suzhouPreferred, + periodMileageKm: vehicle.sourcePeriodMileageKm, + })), + sourceDailyRows: dailySourceRows.length, + expandedDailyRows: dailyRows.length, + missingDailyRows: dailyRows.filter((row) => row.recordStatus === "缺失").length, + customerFilledVehicles: selectedVehicles.filter((row) => row.customerName).length, + }, + null, + 2, + ), +); diff --git a/vehicle-data-platform/apps/api/cmd/daily-geography/main.go b/vehicle-data-platform/apps/api/cmd/daily-geography/main.go new file mode 100644 index 00000000..b9a7e3cf --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/daily-geography/main.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + _ "github.com/go-sql-driver/mysql" + _ "github.com/taosdata/driver-go/v3/taosWS" + "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/dailygeo" + "log" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + if e := run(); e != nil { + log.Fatal(e) + } +} +func run() error { + mode := flag.String("mode", "status", "status, backfill, resolve, or live") + from := flag.String("from", "", "first historical date, defaults to first hydrogen day") + to := flag.String("to", "", "last historical date, defaults to yesterday") + concurrency := flag.Int("concurrency", 3, "maximum concurrent geocoding requests (1-8)") + flag.Parse() + cfg := config.Load() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + db, e := sql.Open("mysql", cfg.MySQLDSN) + if e != nil { + return e + } + defer db.Close() + db.SetMaxOpenConns(12) + if *from == "" { + if e = db.QueryRowContext(ctx, `SELECT COALESCE(DATE_FORMAT(MIN(stat_date),'%Y-%m-%d'),'') FROM vehicle_open_daily_energy WHERE energy_type='HYDROGEN'`).Scan(from); e != nil { + return e + } + } + if *to == "" { + *to = time.Now().In(dailygeo.Shanghai).AddDate(0, 0, -1).Format("2006-01-02") + } + first, e := time.ParseInLocation("2006-01-02", *from, dailygeo.Shanghai) + if e != nil { + return e + } + last, e := time.ParseInLocation("2006-01-02", *to, dailygeo.Shanghai) + if e != nil { + return e + } + if first.After(last) { + return fmt.Errorf("invalid date range") + } + if *mode == "status" { + return printStatus(ctx, db, *from, *to) + } + if cfg.AMapAPIKey == "" { + return fmt.Errorf("AMAP_API_KEY required") + } + td, e := sql.Open(cfg.TDengineDriver, cfg.TDengineDSN) + if e != nil { + return e + } + defer td.Close() + td.SetMaxOpenConns(2) + w := dailygeo.New(db, td, cfg.TDengineDatabase, &dailygeo.AMap{Key: cfg.AMapAPIKey}) + w.Concurrency = *concurrency + if *mode == "live" { + w.Run(ctx) + return ctx.Err() + } + if *mode != "backfill" && *mode != "resolve" { + return fmt.Errorf("invalid mode") + } + return w.WithLock(ctx, "vehicle_daily_geography_backfill", func() error { + if *mode == "backfill" { + for d := first; !d.After(last); d = d.AddDate(0, 0, 1) { + rctx, c := context.WithTimeout(ctx, 90*time.Second) + n, e := w.RefreshDay(rctx, d.Format("2006-01-02")) + c() + if e != nil { + return e + } + log.Printf("daily_geography_seed date=%s points=%d", d.Format("2006-01-02"), n) + } + } + for { + n, failed, e := w.ResolvePending(ctx, *from, *to, 1000) + log.Printf("daily_geography_batch resolved=%d failed=%d error=%v", n, failed, e) + if ctx.Err() != nil { + return ctx.Err() + } + if e != nil && failed == 0 { + return e + } + if n+failed == 0 { + break + } + } + return printStatus(ctx, db, *from, *to) + }) +} +func printStatus(ctx context.Context, db *sql.DB, from, to string) error { + rows, e := db.QueryContext(ctx, `SELECT status,COUNT(*) FROM vehicle_daily_geography WHERE stat_date BETWEEN ? AND ? GROUP BY status`, from, to) + if e != nil { + return e + } + defer rows.Close() + counts := map[string]int{} + for rows.Next() { + var s string + var n int + if e = rows.Scan(&s, &n); e != nil { + return e + } + counts[s] = n + } + if e = rows.Err(); e != nil { + return e + } + rows.Close() + var missing int + e = db.QueryRowContext(ctx, `SELECT COUNT(*) FROM vehicle_open_daily_energy h LEFT JOIN vehicle_daily_geography g ON g.vin=h.vin COLLATE utf8mb4_unicode_ci AND g.stat_date=h.stat_date WHERE h.energy_type='HYDROGEN' AND h.stat_date BETWEEN ? AND ? AND g.vin IS NULL`, from, to).Scan(&missing) + if e != nil { + return e + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"from": from, "to": to, "counts": counts, "hydrogenDaysMissing": missing}) +} diff --git a/vehicle-data-platform/apps/api/cmd/mileage-reconcile-audit/main.go b/vehicle-data-platform/apps/api/cmd/mileage-reconcile-audit/main.go new file mode 100644 index 00000000..24416339 --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/mileage-reconcile-audit/main.go @@ -0,0 +1,93 @@ +// Read-only production verification of the same repository used by HTTP APIs. +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + _ "github.com/go-sql-driver/mysql" + "lingniu/vehicle-data-platform/apps/api/internal/openplatform" + "math" + "os" + "reflect" + "strings" + "time" +) + +func main() { + start := flag.String("start", "2026-09-11", "first date") + end := flag.String("end", "2026-09-15", "last date") + vin := flag.String("vins", "", "VINs, empty = all current vehicles") + flag.Parse() + db, err := sql.Open("mysql", os.Getenv("MYSQL_DSN")) + must(err) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + vins := strings.Split(*vin, ",") + if *vin == "" { + vins = nil + rows, e := db.QueryContext(ctx, "SELECT DISTINCT vin FROM vehicle_identity_binding WHERE vin<>''") + must(e) + for rows.Next() { + var v string + must(rows.Scan(&v)) + vins = append(vins, v) + } + must(rows.Err()) + rows.Close() + } + r := openplatform.NewMySQLRepository(db) + began := time.Now() + priorities := []string{"GB32960", "YUTONG_MQTT"} + all, err := r.ReconciledMileageRange(ctx, vins, *start, *end, priorities) + must(err) + first, e := time.Parse("2006-01-02", *start) + must(e) + last, e := time.Parse("2006-01-02", *end) + must(e) + mismatches := 0 + differences := 0 + normal := 0 + qualities := map[string]int{} + samples := map[string]openplatform.DailyMileage{} + for date := first; !date.After(last); date = date.AddDate(0, 0, 1) { + d := date.Format("2006-01-02") + single, e := r.ReconciledMileageRange(ctx, vins, d, d, priorities) + must(e) + for _, v := range vins { + key := v + "\x00" + d + value, ok := all[key] + one, exists := single[key] + if ok != exists || !reflect.DeepEqual(value, one) { + differences++ + } + if !ok { + continue + } + qualities[value.DataQuality]++ + prev, hasPrev := all[v+"\x00"+date.AddDate(0, 0, -1).Format("2006-01-02")] + if value.DataQuality == "" || value.DataQuality == "CARRIED_FORWARD" { + normal++ + if hasPrev && math.Abs(value.TotalMileageKm-prev.TotalMileageKm-value.MileageKm) > 0.001 { + mismatches++ + } + } + if len(vins) < 10 { + samples[key] = value + } + } + } + must(json.NewEncoder(os.Stdout).Encode(map[string]any{"vehicles": len(vins), "rows": len(all), "normal": normal, "qualities": qualities, "reconciliationMismatches": mismatches, "singleRangeDifferences": differences, "samples": samples, "elapsed": time.Since(began).String()})) + if mismatches > 0 || differences > 0 { + os.Exit(2) + } +} +func must(e error) { + if e != nil { + fmt.Fprintln(os.Stderr, e) + os.Exit(1) + } +} diff --git a/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main.go b/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main.go new file mode 100644 index 00000000..cf7adcd9 --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "flag" + "fmt" + "os" + "time" + + _ "github.com/go-sql-driver/mysql" + "lingniu/vehicle-data-platform/apps/api/internal/platform" +) + +func main() { + apply := flag.Bool("apply", false, "apply corrections after writing a durable backup") + backup := flag.String("backup", "", "new backup JSON path, required with --apply") + flag.Parse() + if *apply && *backup == "" { + panic("--backup is required with --apply") + } + db, err := sql.Open("mysql", os.Getenv("MYSQL_DSN")) + if err != nil { + panic("invalid database configuration") + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err = run(ctx, db, *apply, *backup); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func rows(ctx context.Context, tx *sql.Tx, query string) ([]map[string]any, error) { + r, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer r.Close() + columns, err := r.Columns() + if err != nil { + return nil, err + } + result := []map[string]any{} + for r.Next() { + values := make([]any, len(columns)) + targets := make([]any, len(columns)) + for i := range values { + targets[i] = &values[i] + } + if err = r.Scan(targets...); err != nil { + return nil, err + } + item := map[string]any{} + for i, k := range columns { + if b, ok := values[i].([]byte); ok { + item[k] = string(b) + } else { + item[k] = values[i] + } + } + result = append(result, item) + } + return result, r.Err() +} + +type correction struct { + id, title, status string + remove bool +} + +func run(ctx context.Context, db *sql.DB, apply bool, backup string) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + archive := map[string]any{"createdAt": time.Now().UTC().Format(time.RFC3339), "reason": "decode GB32960 table 18; remove reserved-only false alarms"} + data := map[string][]map[string]any{} + // Consumer must be stopped for --apply. Lock event rows as well so a concurrent + // operator's action is serialized with the correction and fully backed up. + for _, q := range []struct{ key, sql string }{ + {"events", `SELECT * FROM vehicle_alert_event WHERE rule_id='native-gb32960-alarm' ORDER BY id FOR UPDATE`}, + {"evidence", `SELECT n.* FROM vehicle_native_alarm_evidence n JOIN vehicle_alert_event e ON e.id=n.event_id WHERE e.rule_id='native-gb32960-alarm'`}, + {"actions", `SELECT a.* FROM vehicle_alert_event_action a JOIN vehicle_alert_event e ON e.id=a.event_id WHERE e.rule_id='native-gb32960-alarm'`}, + {"state", `SELECT s.* FROM vehicle_native_alarm_state s JOIN vehicle_alert_event e ON e.id=s.active_event_id WHERE e.rule_id='native-gb32960-alarm'`}, + {"notifications", `SELECT n.id,n.event_id FROM vehicle_alert_notification n JOIN vehicle_alert_event e ON e.id=n.event_id WHERE e.rule_id='native-gb32960-alarm'`}, + } { + items, e := rows(ctx, tx, q.sql) + if e != nil { + return e + } + data[q.key] = items + archive[q.key] = items + } + evidence := map[string]string{} + for _, row := range data["evidence"] { + evidence[fmt.Sprint(row["event_id"])] = fmt.Sprint(row["fields_json"]) + } + notified := map[string]bool{} + for _, row := range data["notifications"] { + notified[fmt.Sprint(row["event_id"])] = true + } + changes := []correction{} + renamed, removed := 0, 0 + names := map[string]int{} + for _, row := range data["events"] { + id := fmt.Sprint(row["id"]) + var fields map[string]json.RawMessage + if err = json.Unmarshal([]byte(evidence[id]), &fields); err != nil { + return fmt.Errorf("event %s has no valid evidence; no changes applied", id) + } + description, valid := platform.DescribeNativeAlarm(fields) + if !valid { + return fmt.Errorf("event %s has unsupported evidence; no changes applied", id) + } + names[description.Title]++ + if !description.Active { + if notified[id] { + return fmt.Errorf("event %s has notifications; refusing automatic deletion", id) + } + changes = append(changes, correction{id: id, remove: true}) + removed++ + } else if fmt.Sprint(row["rule_name"]) != description.Title { + changes = append(changes, correction{id: id, title: description.Title, status: fmt.Sprint(row["status"])}) + renamed++ + } + } + if apply && len(changes) > 0 { + file, e := os.OpenFile(backup, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if e != nil { + return e + } + e = json.NewEncoder(file).Encode(archive) + if e == nil { + e = file.Sync() + } + closeErr := file.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + for _, change := range changes { + if change.remove { + if _, err = tx.ExecContext(ctx, `UPDATE vehicle_native_alarm_state SET active_event_id='' WHERE active_event_id=?`, change.id); err != nil { + return err + } + for _, table := range []string{"vehicle_alert_event_action", "vehicle_native_alarm_evidence"} { + if _, err = tx.ExecContext(ctx, "DELETE FROM "+table+" WHERE event_id=?", change.id); err != nil { + return err + } + } + if _, err = tx.ExecContext(ctx, `DELETE FROM vehicle_alert_event WHERE id=? AND rule_id='native-gb32960-alarm'`, change.id); err != nil { + return err + } + } else { + if _, err = tx.ExecContext(ctx, `UPDATE vehicle_alert_event SET rule_name=?,version=version+1 WHERE id=? AND rule_id='native-gb32960-alarm'`, change.title, change.id); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'repair',?,?,'native-alarm-repair','按GB/T 32960.3-2016表18修正具体告警名称，保留原始证据')`, change.id, change.status, change.status); err != nil { + return err + } + } + } + if err = tx.Commit(); err != nil { + return err + } + } + return json.NewEncoder(os.Stdout).Encode(map[string]any{"applied": apply, "scanned": len(data["events"]), "renamed": renamed, "removed": removed, "descriptions": names}) +} diff --git a/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main_test.go b/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main_test.go new file mode 100644 index 00000000..50d79e73 --- /dev/null +++ b/vehicle-data-platform/apps/api/cmd/native-alarm-repair/main_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func expectSnapshot(mock sqlmock.Sqlmock, notified bool) { + mock.ExpectBegin() + mock.ExpectQuery(`SELECT \* FROM vehicle_alert_event`).WillReturnRows(sqlmock.NewRows([]string{"id", "rule_name", "status"}).AddRow("false-event", "车辆原生告警", "unprocessed").AddRow("real-event", "车辆原生告警", "processing")) + payload := func(flag string) string { + data := map[string]any{"gb32960.alarm.max_alarm_level": "0", "gb32960.alarm.general_alarm_flag": flag} + for _, key := range []string{"battery_faults", "motor_faults", "engine_faults", "other_faults"} { + data["gb32960.alarm."+key] = []string{} + } + b, _ := json.Marshal(data) + return string(b) + } + mock.ExpectQuery(`SELECT n.\* FROM vehicle_native_alarm_evidence`).WillReturnRows(sqlmock.NewRows([]string{"event_id", "fields_json"}).AddRow("false-event", payload("0x00300000")).AddRow("real-event", payload("0x00380800"))) + mock.ExpectQuery(`SELECT a.\* FROM vehicle_alert_event_action`).WillReturnRows(sqlmock.NewRows([]string{"event_id", "action"}).AddRow("false-event", "trigger")) + mock.ExpectQuery(`SELECT s.\* FROM vehicle_native_alarm_state`).WillReturnRows(sqlmock.NewRows([]string{"vin", "active_event_id"}).AddRow("VIN1", "false-event")) + notifications := sqlmock.NewRows([]string{"id", "event_id"}) + if notified { + notifications.AddRow(1, "false-event") + } + mock.ExpectQuery(`SELECT n.id,n.event_id`).WillReturnRows(notifications) +} + +func TestRepairDryRunNeverWrites(t *testing.T) { + db, mock, _ := sqlmock.New() + defer db.Close() + expectSnapshot(mock, false) + mock.ExpectRollback() + if err := run(t.Context(), db, false, ""); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} +func TestRepairBacksUpAndCorrectsOnlyNativeEvents(t *testing.T) { + db, mock, _ := sqlmock.New() + defer db.Close() + expectSnapshot(mock, false) + mock.ExpectExec(`UPDATE vehicle_native_alarm_state`).WithArgs("false-event").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DELETE FROM vehicle_alert_event_action`).WithArgs("false-event").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DELETE FROM vehicle_native_alarm_evidence`).WithArgs("false-event").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DELETE FROM vehicle_alert_event WHERE id=\? AND rule_id='native-gb32960-alarm'`).WithArgs("false-event").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`UPDATE vehicle_alert_event SET rule_name=\?,version=version\+1 WHERE id=\? AND rule_id='native-gb32960-alarm'`).WithArgs("绝缘报警", "real-event").WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO vehicle_alert_event_action`).WithArgs("real-event", "processing", "processing").WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectCommit() + path := filepath.Join(t.TempDir(), "backup.json") + if err := run(t.Context(), db, true, path); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var archive map[string]json.RawMessage + if err = json.Unmarshal(data, &archive); err != nil { + t.Fatal(err) + } + for _, key := range []string{"events", "evidence", "actions", "state"} { + if len(archive[key]) == 0 { + t.Fatalf("missing backup %s", key) + } + } + info, _ := os.Stat(path) + if info.Mode().Perm() != 0600 { + t.Fatal("backup permissions") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} +func TestRepairRefusesToOverwriteBackup(t *testing.T) { + db, mock, _ := sqlmock.New() + defer db.Close() + expectSnapshot(mock, false) + mock.ExpectRollback() + path := filepath.Join(t.TempDir(), "backup.json") + if err := os.WriteFile(path, []byte("previous backup"), 0600); err != nil { + t.Fatal(err) + } + if err := run(t.Context(), db, true, path); err == nil { + t.Fatal("overwrote backup") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} +func TestRepairRefusesToOrphanNotifications(t *testing.T) { + db, mock, _ := sqlmock.New() + defer db.Close() + expectSnapshot(mock, true) + mock.ExpectRollback() + if err := run(t.Context(), db, true, filepath.Join(t.TempDir(), "backup.json")); err == nil { + t.Fatal("deleted notified event") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/vehicle-data-platform/apps/api/internal/app/server.go b/vehicle-data-platform/apps/api/internal/app/server.go index fda8f55c..0f6c6f10 100644 --- a/vehicle-data-platform/apps/api/internal/app/server.go +++ b/vehicle-data-platform/apps/api/internal/app/server.go @@ -19,6 +19,7 @@ import ( "time" "lingniu/vehicle-data-platform/apps/api/internal/config" + "lingniu/vehicle-data-platform/apps/api/internal/dailygeo" "lingniu/vehicle-data-platform/apps/api/internal/httpx" "lingniu/vehicle-data-platform/apps/api/internal/openplatform" "lingniu/vehicle-data-platform/apps/api/internal/platform" @@ -69,6 +70,11 @@ func NewServer(cfg config.Config) http.Handler { log.Printf("production capacity-check probe enabled") } productionStore.WithAlertStreamConfig(cfg.AlertStreamMode, cfg.AlertStreamKafkaGroup) + if cfg.DailyGeographyEnabled && tdengine != nil && cfg.AMapAPIKey != "" { + geographyWorker := dailygeo.New(db, tdengine, cfg.TDengineDatabase, &dailygeo.AMap{Key: cfg.AMapAPIKey}) + geographyWorker.RequestInterval = 100 * time.Millisecond + go geographyWorker.Run(context.Background()) + } store = productionStore storeErr = nil log.Printf("production mysql store enabled") diff --git a/vehicle-data-platform/apps/api/internal/config/config.go b/vehicle-data-platform/apps/api/internal/config/config.go index 0a9e859e..6f5c123e 100644 --- a/vehicle-data-platform/apps/api/internal/config/config.go +++ b/vehicle-data-platform/apps/api/internal/config/config.go @@ -8,6 +8,7 @@ import ( ) type Config struct { + DailyGeographyEnabled bool HTTPAddr string StaticDir string MySQLDSN string @@ -97,6 +98,7 @@ func Load() Config { RequestTimeout: time.Duration(envInt("REQUEST_TIMEOUT_MS", 5000)) * time.Millisecond, AMapWebJSKey: os.Getenv("AMAP_WEB_JS_KEY"), AMapAPIKey: os.Getenv("AMAP_API_KEY"), + DailyGeographyEnabled: envBool("DAILY_GEOGRAPHY_ENABLED", false), AMapSecurityCode: os.Getenv("AMAP_SECURITY_JS_CODE"), AMapServiceHost: os.Getenv("AMAP_SECURITY_SERVICE_HOST"), PlatformRelease: os.Getenv("PLATFORM_RELEASE"), diff --git a/vehicle-data-platform/apps/api/internal/dailygeo/coordinates.go b/vehicle-data-platform/apps/api/internal/dailygeo/coordinates.go new file mode 100644 index 00000000..593bf374 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/dailygeo/coordinates.go @@ -0,0 +1,35 @@ +package dailygeo + +import "math" + +func WGS84ToGCJ02(longitude float64, latitude float64) (float64, float64) { + if longitude < 72.004 || longitude > 137.8347 || latitude < 0.8293 || latitude > 55.8271 { + return longitude, latitude + } + const semiMajorAxis = 6378245.0 + const eccentricitySquared = 0.006693421622965943 + longitudeOffset := transformGCJLongitude(longitude-105, latitude-35) + latitudeOffset := transformGCJLatitude(longitude-105, latitude-35) + radianLatitude := latitude / 180 * math.Pi + magic := 1 - eccentricitySquared*math.Pow(math.Sin(radianLatitude), 2) + squareRootMagic := math.Sqrt(magic) + convertedLatitude := latitude + latitudeOffset*180/((semiMajorAxis*(1-eccentricitySquared))/(magic*squareRootMagic)*math.Pi) + convertedLongitude := longitude + longitudeOffset*180/(semiMajorAxis/squareRootMagic*math.Cos(radianLatitude)*math.Pi) + return convertedLongitude, convertedLatitude +} + +func transformGCJLatitude(longitude float64, latitude float64) float64 { + value := -100 + 2*longitude + 3*latitude + 0.2*latitude*latitude + 0.1*longitude*latitude + 0.2*math.Sqrt(math.Abs(longitude)) + value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3 + value += (20*math.Sin(latitude*math.Pi) + 40*math.Sin(latitude/3*math.Pi)) * 2 / 3 + value += (160*math.Sin(latitude/12*math.Pi) + 320*math.Sin(latitude*math.Pi/30)) * 2 / 3 + return value +} + +func transformGCJLongitude(longitude float64, latitude float64) float64 { + value := 300 + longitude + 2*latitude + 0.1*longitude*longitude + 0.1*longitude*latitude + 0.1*math.Sqrt(math.Abs(longitude)) + value += (20*math.Sin(6*longitude*math.Pi) + 20*math.Sin(2*longitude*math.Pi)) * 2 / 3 + value += (20*math.Sin(longitude*math.Pi) + 40*math.Sin(longitude/3*math.Pi)) * 2 / 3 + value += (150*math.Sin(longitude/12*math.Pi) + 300*math.Sin(longitude/30*math.Pi)) * 2 / 3 + return value +} diff --git a/vehicle-data-platform/apps/api/internal/dailygeo/geocoder.go b/vehicle-data-platform/apps/api/internal/dailygeo/geocoder.go new file mode 100644 index 00000000..9195deea --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/dailygeo/geocoder.go @@ -0,0 +1,123 @@ +package dailygeo + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type Address struct{ Province, City, Region, Adcode string } +type Resolver interface { + Resolve(context.Context, float64, float64) (Address, error) +} +type AMap struct { + Key, BaseURL string + Client *http.Client +} + +func (a *AMap) Resolve(ctx context.Context, lng, lat float64) (Address, error) { + if a.Key == "" { + return Address{}, fmt.Errorf("AMAP_API_KEY is not configured") + } + base := a.BaseURL + if base == "" { + base = "https://restapi.amap.com" + } + u, e := url.Parse(base + "/v3/geocode/regeo") + if e != nil { + return Address{}, fmt.Errorf("invalid geocoder URL") + } + x, y := WGS84ToGCJ02(lng, lat) + q := u.Query() + q.Set("key", a.Key) + q.Set("location", fmt.Sprintf("%.6f,%.6f", x, y)) + q.Set("extensions", "base") + q.Set("radius", "1000") + u.RawQuery = q.Encode() + req, e := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if e != nil { + return Address{}, fmt.Errorf("invalid geocoder request") + } + client := a.Client + if client == nil { + client = &http.Client{Timeout: 10 * time.Second} + } + resp, e := client.Do(req) + if e != nil { + return Address{}, fmt.Errorf("geocoder request failed") + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return Address{}, fmt.Errorf("geocoder HTTP %d", resp.StatusCode) + } + var body struct { + Status string + Infocode string + Regeocode struct { + AddressComponent struct{ Province, City, District, Adcode json.RawMessage } `json:"addressComponent"` + } + } + if e = json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&body); e != nil { + return Address{}, fmt.Errorf("invalid geocoder response") + } + if body.Status != "1" { + return Address{}, fmt.Errorf("geocoder rejected: %s", body.Infocode) + } + p, c := textValue(body.Regeocode.AddressComponent.Province), textValue(body.Regeocode.AddressComponent.City) + if c == "" { + switch p { + case "北京市", "上海市", "天津市", "重庆市", "香港特别行政区", "澳门特别行政区": + c = p + } + } + // Province-administered county-level cities (for example 潜江市) are + // returned in district, with city=[], by AMap. + if c == "" && p != "" { + district := textValue(body.Regeocode.AddressComponent.District) + if strings.HasSuffix(district, "市") { + c = district + } + } + if p == "" || c == "" { + return Address{}, fmt.Errorf("geocoder returned incomplete province/city") + } + return Address{p, c, Region(p), textValue(body.Regeocode.AddressComponent.Adcode)}, nil +} +func textValue(raw json.RawMessage) string { + var s string + if json.Unmarshal(raw, &s) == nil { + return s + } + var ss []string + if json.Unmarshal(raw, &ss) == nil { + return strings.Join(ss, "") + } + return "" +} +func Region(province string) string { + groups := []struct { + name string + provinces []string + }{ + {"华东", []string{"上海", "江苏", "浙江", "安徽", "福建", "江西", "山东", "台湾"}}, + {"华北", []string{"北京", "天津", "河北", "山西", "内蒙古"}}, + {"华中", []string{"河南", "湖北", "湖南"}}, + {"华南", []string{"广东", "广西", "海南", "香港", "澳门"}}, + {"东北", []string{"辽宁", "吉林", "黑龙江"}}, + {"西南", []string{"重庆", "四川", "贵州", "云南", "西藏"}}, + {"西北", []string{"陕西", "甘肃", "青海", "宁夏", "新疆"}}, + } + for _, g := range groups { + for _, p := range g.provinces { + if strings.HasPrefix(province, p) { + return g.name + } + } + } + return "" +} diff --git a/vehicle-data-platform/apps/api/internal/dailygeo/worker.go b/vehicle-data-platform/apps/api/internal/dailygeo/worker.go new file mode 100644 index 00000000..b7cf3f93 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/dailygeo/worker.go @@ -0,0 +1,297 @@ +package dailygeo + +import ( + "context" + "database/sql" + "fmt" + "log" + "math" + "regexp" + "sync" + "time" +) + +var Shanghai = time.FixedZone("Asia/Shanghai", 8*3600) +var identifier = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +type Point struct { + VIN, Protocol string + Time time.Time + Longitude, Latitude float64 +} +type Worker struct { + RequestInterval time.Duration + DB, TD *sql.DB + Database string + Resolver Resolver + Concurrency int + requestMu sync.Mutex + nextRequest time.Time +} + +func New(db, td *sql.DB, database string, r Resolver) *Worker { + return &Worker{DB: db, TD: td, Database: database, Resolver: r, Concurrency: 3} +} +func validPoint(p Point) bool { + return p.VIN != "" && !(p.Longitude == -0.999999 && p.Latitude == -0.999999) && !p.Time.IsZero() && p.Longitude != 0 && p.Latitude != 0 && !math.IsNaN(p.Longitude) && !math.IsNaN(p.Latitude) && math.Abs(p.Longitude) <= 180 && math.Abs(p.Latitude) <= 90 +} +func newer(a, b Point) bool { + return a.Time.After(b.Time) || (a.Time.Equal(b.Time) && a.Protocol < b.Protocol) +} + +// Last() is applied only to non-null valid coordinates, partitioned by VIN and +// protocol. All selected columns therefore refer to the same final source row. +func (w *Worker) RefreshDay(ctx context.Context, date string) (int, error) { + day, e := time.ParseInLocation("2006-01-02", date, Shanghai) + if e != nil { + return 0, e + } + if w.TD == nil || !identifier.MatchString(w.Database) { + return 0, fmt.Errorf("historical location store unavailable") + } + q := fmt.Sprintf(`SELECT vin,protocol,LAST(ts),LAST(longitude),LAST(latitude) FROM %s.vehicle_locations WHERE ts >= '%s' AND ts < '%s' AND NOT (longitude = -0.999999 AND latitude = -0.999999) AND longitude <> 0 AND longitude BETWEEN -180 AND 180 AND latitude <> 0 AND latitude BETWEEN -90 AND 90 PARTITION BY vin,protocol`, w.Database, day.Format(time.RFC3339), day.AddDate(0, 0, 1).Format(time.RFC3339)) + rows, e := w.TD.QueryContext(ctx, q) + if e != nil { + return 0, e + } + points := map[string]Point{} + for rows.Next() { + var p Point + if e = rows.Scan(&p.VIN, &p.Protocol, &p.Time, &p.Longitude, &p.Latitude); e != nil { + rows.Close() + return 0, e + } + if !validPoint(p) || p.Time.In(Shanghai).Format("2006-01-02") != date { + continue + } + if old, ok := points[p.VIN]; !ok || newer(p, old) { + points[p.VIN] = p + } + } + e = rows.Err() + rows.Close() + if e != nil { + return 0, e + } + // Repair the device's invalid-location sentinel before applying real points. + // Clearing its timestamp allows an earlier valid point from this day to win. + _, e = w.DB.ExecContext(ctx, `UPDATE vehicle_daily_geography SET longitude=NULL,latitude=NULL,location_time=NULL,province='',city='',region='',adcode='',status='NO_LOCATION',attempts=0,last_error='',next_attempt_at=NULL,resolved_at=NULL WHERE stat_date=? AND longitude=-0.999999 AND latitude=-0.999999`, date) + if e != nil { + return 0, e + } + for _, p := range points { + if e = w.SavePoint(ctx, p); e != nil { + return 0, e + } + } + // A successful historical lookup with no point is distinct from an unavailable + // history service. Never record NO_LOCATION when the lookup itself failed. + _, e = w.DB.ExecContext(ctx, `INSERT IGNORE INTO vehicle_daily_geography(vin,stat_date,status) SELECT vin,stat_date,'NO_LOCATION' FROM vehicle_open_daily_energy WHERE energy_type='HYDROGEN' AND stat_date=?`, date) + return len(points), e +} + +const savePointSQL = `INSERT INTO vehicle_daily_geography(vin,stat_date,longitude,latitude,location_time,source_protocol,status) VALUES(?,?,?,?,?,?,'PENDING') ON DUPLICATE KEY UPDATE + province=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'',province), + city=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'',city), + region=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'',region), + adcode=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'',adcode), + status=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'PENDING',status), + attempts=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),0,attempts), + next_attempt_at=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),NULL,next_attempt_at), + last_error=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),'',last_error), + resolved_at=IF((location_time IS NULL OR VALUES(location_time)>location_time) AND (longitude IS NULL OR longitude<>VALUES(longitude) OR latitude<>VALUES(latitude)),NULL,resolved_at), + longitude=IF(location_time IS NULL OR VALUES(location_time)>location_time,VALUES(longitude),longitude), + latitude=IF(location_time IS NULL OR VALUES(location_time)>location_time,VALUES(latitude),latitude), + source_protocol=IF(location_time IS NULL OR VALUES(location_time)>location_time,VALUES(source_protocol),source_protocol), + location_time=IF(location_time IS NULL OR VALUES(location_time)>location_time,VALUES(location_time),location_time)` + +func (w *Worker) SavePoint(ctx context.Context, p Point) error { + if !validPoint(p) { + return fmt.Errorf("invalid historical location") + } + _, e := w.DB.ExecContext(ctx, savePointSQL, p.VIN, p.Time.In(Shanghai).Format("2006-01-02"), math.Round(p.Longitude*1e6)/1e6, math.Round(p.Latitude*1e6)/1e6, p.Time.In(Shanghai), p.Protocol) + return e +} + +type pending struct { + VIN, Date string + Longitude, Latitude float64 +} + +func (w *Worker) ResolvePending(ctx context.Context, from, to string, limit int) (int, int, error) { + if limit <= 0 || limit > 5000 { + limit = 500 + } + rows, e := w.DB.QueryContext(ctx, `SELECT vin,DATE_FORMAT(stat_date,'%Y-%m-%d'),longitude,latitude FROM vehicle_daily_geography WHERE stat_date BETWEEN ? AND ? AND status IN ('PENDING','ERROR') AND (next_attempt_at IS NULL OR next_attempt_at<=NOW(3)) ORDER BY stat_date DESC,vin LIMIT ?`, from, to, limit) + if e != nil { + return 0, 0, e + } + items := []pending{} + for rows.Next() { + var p pending + if e = rows.Scan(&p.VIN, &p.Date, &p.Longitude, &p.Latitude); e != nil { + rows.Close() + return 0, 0, e + } + items = append(items, p) + } + e = rows.Err() + rows.Close() + if e != nil { + return 0, 0, e + } + jobs := make(chan pending, len(items)) + for _, p := range items { + jobs <- p + } + close(jobs) + count := w.Concurrency + if count < 1 { + count = 1 + } + if count > 8 { + count = 8 + } + var mu sync.Mutex + var wg sync.WaitGroup + ok, failed := 0, 0 + var first error + for i := 0; i < count; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for p := range jobs { + if ctx.Err() != nil { + return + } + e := w.resolveOne(ctx, p) + mu.Lock() + if e == nil { + ok++ + } else { + failed++ + if first == nil { + first = e + } + } + mu.Unlock() + } + }() + } + wg.Wait() + if ctx.Err() != nil { + return ok, failed, ctx.Err() + } + return ok, failed, first +} +func (w *Worker) resolveOne(ctx context.Context, p pending) error { + var a Address + e := w.DB.QueryRowContext(ctx, `SELECT province,city,region,adcode FROM vehicle_geography_cache WHERE longitude=? AND latitude=? AND resolved_at>=DATE_SUB(NOW(),INTERVAL 180 DAY)`, p.Longitude, p.Latitude).Scan(&a.Province, &a.City, &a.Region, &a.Adcode) + if e != nil && e != sql.ErrNoRows { + return e + } + if e == sql.ErrNoRows { + // At most three external calls in flight by default; quota/network failures + // are persisted and retried with backoff, never exported as a guessed city. + w.requestMu.Lock() + delay := time.Until(w.nextRequest) + if delay > 0 { + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + w.requestMu.Unlock() + return ctx.Err() + case <-timer.C: + } + } + interval := w.RequestInterval + if interval <= 0 { + interval = 20 * time.Millisecond + } + w.nextRequest = time.Now().Add(interval) + w.requestMu.Unlock() + rctx, cancel := context.WithTimeout(ctx, 12*time.Second) + a, e = w.Resolver.Resolve(rctx, p.Longitude, p.Latitude) + cancel() + if e != nil { + _, saveErr := w.DB.ExecContext(ctx, `UPDATE vehicle_daily_geography SET status='ERROR',attempts=attempts+1,last_error=?,next_attempt_at=DATE_ADD(NOW(3),INTERVAL LEAST(3600,60*POW(2,LEAST(attempts,6))) SECOND) WHERE vin=? AND stat_date=? AND longitude=? AND latitude=?`, e.Error(), p.VIN, p.Date, p.Longitude, p.Latitude) + if saveErr != nil { + return saveErr + } + return e + } + _, e = w.DB.ExecContext(ctx, `INSERT INTO vehicle_geography_cache(longitude,latitude,province,city,region,adcode) VALUES(?,?,?,?,?,?) ON DUPLICATE KEY UPDATE province=VALUES(province),city=VALUES(city),region=VALUES(region),adcode=VALUES(adcode),resolved_at=NOW(3)`, p.Longitude, p.Latitude, a.Province, a.City, a.Region, a.Adcode) + if e != nil { + return e + } + } + _, e = w.DB.ExecContext(ctx, `UPDATE vehicle_daily_geography SET province=?,city=?,region=?,adcode=?,status='RESOLVED',last_error='',attempts=0,next_attempt_at=NULL,resolved_at=NOW(3) WHERE vin=? AND stat_date=? AND longitude=? AND latitude=?`, a.Province, a.City, a.Region, a.Adcode, p.VIN, p.Date, p.Longitude, p.Latitude) + return e +} + +// An advisory lock is tied to this dedicated connection and released even on +// cancellation/crash, preventing multiple API replicas from running the loop. +func (w *Worker) WithLock(ctx context.Context, name string, fn func() error) error { + conn, e := w.DB.Conn(ctx) + if e != nil { + return e + } + defer conn.Close() + var acquired int + if e = conn.QueryRowContext(ctx, `SELECT GET_LOCK(?,0)`, name).Scan(&acquired); e != nil { + return e + } + if acquired != 1 { + return fmt.Errorf("daily geography worker already running") + } + defer conn.ExecContext(context.Background(), `SELECT RELEASE_LOCK(?)`, name) + return fn() +} +func (w *Worker) Run(ctx context.Context) { + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + round := 0 + for { + e := w.WithLock(ctx, "vehicle_daily_geography_live", func() error { + today := time.Now().In(Shanghai) + date := today.Format("2006-01-02") + rctx, c := context.WithTimeout(ctx, 50*time.Second) + defer c() + if _, e := w.RefreshDay(rctx, date); e != nil { + return e + } + if round%60 == 0 { + for d := 1; d <= 2; d++ { + if _, e := w.RefreshDay(rctx, today.AddDate(0, 0, -d).Format("2006-01-02")); e != nil { + return e + } + } + } + n, failed, e := w.ResolvePending(rctx, today.AddDate(0, 0, -2).Format("2006-01-02"), date, 500) + log.Printf("daily_geography resolved=%d failed=%d", n, failed) + if e != nil { + return e + } + // Historical retry is independent of the live queue. The same lock used + // by the CLI prevents duplicate external calls during a bulk backfill. + _ = w.WithLock(rctx, "vehicle_daily_geography_backfill", func() error { + n, failed, e := w.ResolvePending(rctx, "2000-01-01", today.AddDate(0, 0, -3).Format("2006-01-02"), 200) + if n+failed > 0 { + log.Printf("daily_geography_history_retry resolved=%d failed=%d error=%v", n, failed, e) + } + return e + }) + return nil + }) + if e != nil { + log.Printf("daily_geography: %v", e) + } + round++ + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} diff --git a/vehicle-data-platform/apps/api/internal/dailygeo/worker_test.go b/vehicle-data-platform/apps/api/internal/dailygeo/worker_test.go new file mode 100644 index 00000000..9171fbc3 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/dailygeo/worker_test.go @@ -0,0 +1,138 @@ +package dailygeo + +import ( + "context" + "database/sql" + "errors" + "github.com/DATA-DOG/go-sqlmock" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "testing" + "time" +) + +type resolverFunc func(context.Context, float64, float64) (Address, error) + +func (f resolverFunc) Resolve(c context.Context, x, y float64) (Address, error) { return f(c, x, y) } +func TestAMapMunicipalityAndRegion(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("key") != "test-key" { + t.Error("missing credential") + } + w.Write([]byte(`{"status":"1","regeocode":{"addressComponent":{"province":"上海市","city":[],"adcode":"310101"}}}`)) + })) + defer server.Close() + a, e := (&AMap{Key: "test-key", BaseURL: server.URL}).Resolve(context.Background(), 121.47, 31.23) + if e != nil || a.Province != "上海市" || a.City != "上海市" || a.Region != "华东" { + t.Fatalf("address=%+v error=%v", a, e) + } + for p, want := range map[string]string{"广东省": "华南", "四川省": "西南", "陕西省": "西北", "山西省": "华北", "湖北省": "华中", "吉林省": "东北", "": ""} { + if Region(p) != want { + t.Fatal(p) + } + } +} +func TestAMapQuotaFailureDoesNotLeakKey(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"status":"0","infocode":"10021"}`)) })) + defer server.Close() + _, e := (&AMap{Key: "secret-value", BaseURL: server.URL}).Resolve(context.Background(), 121, 31) + if e == nil || !strings.Contains(e.Error(), "10021") || strings.Contains(e.Error(), "secret-value") { + t.Fatal(e) + } +} +func TestCachedCoordinateNeedsNoGeocoding(t *testing.T) { + db, m, e := sqlmock.New() + if e != nil { + t.Fatal(e) + } + defer db.Close() + m.ExpectQuery("SELECT province,city,region,adcode").WithArgs(121.0, 31.0).WillReturnRows(sqlmock.NewRows([]string{"province", "city", "region", "adcode"}).AddRow("上海市", "上海市", "华东", "310000")) + m.ExpectExec("UPDATE vehicle_daily_geography SET province=.*WHERE vin=\\? AND stat_date=\\? AND longitude=\\? AND latitude=\\?").WithArgs("上海市", "上海市", "华东", "310000", "VIN1", "2026-08-31", 121.0, 31.0).WillReturnResult(sqlmock.NewResult(0, 1)) + w := New(db, nil, "", resolverFunc(func(context.Context, float64, float64) (Address, error) { + t.Fatal("cache hit must not call external API") + return Address{}, nil + })) + if e = w.resolveOne(context.Background(), pending{"VIN1", "2026-08-31", 121, 31}); e != nil { + t.Fatal(e) + } + if e = m.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } +} +func TestResolverFailurePersistsRetryInsteadOfFalseUnknown(t *testing.T) { + db, m, _ := sqlmock.New() + defer db.Close() + m.ExpectQuery("SELECT province,city,region,adcode").WillReturnError(sql.ErrNoRows) + m.ExpectExec("UPDATE vehicle_daily_geography SET status='ERROR'.*next_attempt_at=.*WHERE vin=\\? AND stat_date=\\? AND longitude=\\? AND latitude=\\?").WithArgs("quota", "VIN1", "2026-08-31", 121.0, 31.0).WillReturnResult(sqlmock.NewResult(0, 1)) + w := New(db, nil, "", resolverFunc(func(context.Context, float64, float64) (Address, error) { return Address{}, errors.New("quota") })) + if e := w.resolveOne(context.Background(), pending{"VIN1", "2026-08-31", 121, 31}); e == nil { + t.Fatal("expected failure") + } + if e := m.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } +} +func TestHistoricalFailureDoesNotMarkMissingLocation(t *testing.T) { + db, m, _ := sqlmock.New() + defer db.Close() + td, tm, _ := sqlmock.New() + defer td.Close() + tm.ExpectQuery("SELECT vin,protocol,LAST").WillReturnError(errors.New("historical store unavailable")) + w := New(db, td, "test_ts", nil) + if _, e := w.RefreshDay(context.Background(), "2026-08-31"); e == nil { + t.Fatal("expected history error") + } + if e := m.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } + if e := tm.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } +} +func TestHistoryUsesLatestSourceInsideBusinessDate(t *testing.T) { + db, m, _ := sqlmock.New() + defer db.Close() + td, tm, _ := sqlmock.New() + defer td.Close() + early := time.Date(2026, 8, 31, 2, 0, 0, 0, Shanghai) + late := early.Add(20 * time.Hour) + tm.ExpectQuery("SELECT vin,protocol,LAST.*2026-08-31T00:00:00\\+08:00.*2026-09-01T00:00:00\\+08:00.*PARTITION BY vin,protocol").WillReturnRows(sqlmock.NewRows([]string{"vin", "protocol", "ts", "longitude", "latitude"}).AddRow("VIN1", "GB32960", late, 121.0, 31.0).AddRow("VIN1", "JT808", early, 113.0, 23.0).AddRow("VIN2", "JT808", late.Add(5*time.Hour), 110.0, 20.0)) + m.ExpectExec("UPDATE vehicle_daily_geography SET longitude=NULL").WithArgs("2026-08-31").WillReturnResult(sqlmock.NewResult(0, 1)) + m.ExpectExec(regexp.QuoteMeta(savePointSQL)).WithArgs("VIN1", "2026-08-31", 121.0, 31.0, late, "GB32960").WillReturnResult(sqlmock.NewResult(0, 1)) + m.ExpectExec("INSERT IGNORE INTO vehicle_daily_geography").WithArgs("2026-08-31").WillReturnResult(sqlmock.NewResult(0, 0)) + w := New(db, td, "test_ts", nil) + n, e := w.RefreshDay(context.Background(), "2026-08-31") + if e != nil || n != 1 { + t.Fatalf("n=%d error=%v", n, e) + } + if e = m.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } + if e = tm.ExpectationsWereMet(); e != nil { + t.Fatal(e) + } +} + +func TestProvinceAdministeredCity(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"status":"1","regeocode":{"addressComponent":{"province":"湖北省","city":[],"district":"潜江市","adcode":"429005"}}}`)) + })) + defer server.Close() + a, e := (&AMap{Key: "test", BaseURL: server.URL}).Resolve(context.Background(), 112.807848, 30.381828) + if e != nil || a.City != "潜江市" || a.Region != "华中" { + t.Fatalf("address=%+v error=%v", a, e) + } +} + +func TestInvalidLocationSentinel(t *testing.T) { + p := Point{VIN: "VIN1", Time: time.Now(), Longitude: -0.999999, Latitude: -0.999999} + if validPoint(p) { + t.Fatal("invalid device sentinel accepted") + } + p.Longitude, p.Latitude = 114.3, 30.5 + if !validPoint(p) { + t.Fatal("valid location rejected") + } +} diff --git a/vehicle-data-platform/apps/api/internal/openplatform/assets/docs.html b/vehicle-data-platform/apps/api/internal/openplatform/assets/docs.html index e7959ecb..34d15350 100644 --- a/vehicle-data-platform/apps/api/internal/openplatform/assets/docs.html +++ b/vehicle-data-platform/apps/api/internal/openplatform/assets/docs.html @@ -81,7 +81,7 @@

车辆单日用氢量

POST/api/v1/vehicles/hydrogen-consumption/query

查询车辆单日用氢量，单位 kg。NORMAL + OK + PRELIMINARY 仅供初步监控；正式报表要求 FINAL 并核对证据与版本。两个日统计接口没有共同快照，区间缺失或不同不能直接计算百公里氢耗。

#

请求参数

字段必填说明
date是日期，格式 yyyy-MM-dd
plateNumbers否车牌数组；省略或 [] 时查询全部授权车辆

成功返回 · 200

{
  "code": "SUCCESS",
  "data": [{
    "plateNumber": "浙F06618F",
    "hydrogenConsumptionKg": 12.315,
    "status": "NORMAL"
  }]
}
  • 400：日期或车牌格式不正确
  • 401：appKey 无效、停用或过期
  • 403：指定车辆未授权
-

车辆单日里程

POST/api/v1/vehicles/mileage/query

返回当日行驶里程、当日累计总里程和实际选用的数据协议，单位 km。

#

请求参数

字段必填说明
date是日期，格式 yyyy-MM-dd
plateNumbers否省略时查询全部授权车辆
protocolPriority否协议选源顺序，例如 ["GB32960","MQTT","JT808"]
缺数规则：若当日没有有效里程，日里程为 0；累计总里程沿用上一个有效统计周期，计算时间也显示该周期时间。

成功返回 · 200

{
  "code": "SUCCESS",
  "data": [{
    "dailyMileageKm": 182.437,
    "totalMileageKm": 12345.679,
    "sourceProtocol": "GB32960",
    "status": "NORMAL"
  }]
}
  • 400：日期、车牌或协议参数错误
  • 403：授权期未覆盖查询日
  • 无统计：单车以 NO_DATA 返回
+

车辆单日里程

POST/api/v1/vehicles/mileage/query

返回当日行驶里程、当日累计总里程和实际选用的数据协议，单位 km。

#

请求参数

字段必填说明
date是日期，格式 yyyy-MM-dd
plateNumbers否省略时查询全部授权车辆
protocolPriority否协议选源顺序，例如 ["GB32960","MQTT","JT808"]
缺数规则：日里程按同来源相邻日累计差计算；缺报日沿用累计值、日里程补 0（CARRIED_FORWARD），跨缺报期增量计入恢复日。首次基线、来源切换或累计回退返回 DATA_ANOMALY，日里程为 null。GPS 估算不参与本接口。

成功返回 · 200

{
  "code": "SUCCESS",
  "data": [{
    "dailyMileageKm": 182.437,
    "totalMileageKm": 12345.679,
    "sourceProtocol": "GB32960",
    "status": "NORMAL"
  }]
}
  • 400：日期、车牌或协议参数错误
  • 403：授权期未覆盖查询日
  • 无统计：单车以 NO_DATA 返回

车辆区间日里程

POST/api/v1/vehicles/mileage/range/query

按车辆、日期分页返回区间日里程，最长查询区间为 366 天。

#

请求参数

字段必填说明
startDate / endDate是日期区间，yyyy-MM-dd
plateNumbers否车牌数组，最多 5000 辆
protocolPriority否协议选源顺序
pageSize / cursor否分页大小及下一页游标

成功返回 · 200

{
  "code": "SUCCESS",
  "data": [{
    "date": "2026-08-06",
    "dailyMileageKm": 182.437,
    "totalMileageKm": 12345.679
  }],
  "nextCursor": null
}
  • 400：区间超限或游标与原参数不一致
  • 403：授权未完整覆盖查询区间
diff --git a/vehicle-data-platform/apps/api/internal/openplatform/assets/openapi.yaml b/vehicle-data-platform/apps/api/internal/openplatform/assets/openapi.yaml index 08dfe1a2..64eca0e4 100644 --- a/vehicle-data-platform/apps/api/internal/openplatform/assets/openapi.yaml +++ b/vehicle-data-platform/apps/api/internal/openplatform/assets/openapi.yaml @@ -102,9 +102,9 @@ paths: summary: 查询车辆单日里程 description: | plateNumbers 省略或传空数组时，返回该应用在查询自然日有效授权的全部车辆。 - protocolPriority 传入时，逐车按数组顺序选择第一个有效协议，未列出的协议完全禁用且不会兜底；省略时保持平台默认选源行为。 - NORMAL 结果同时包含日里程、日末累计总里程、实际来源协议、源数据时间和投影更新时间。日里程可由 GPS 轨迹估算；累计总里程读取每日统计中的 day_end_total_mileage_km，始终优先采用同协议终端上报的累计里程，不会使用 GPS 日里程估算值冒充累计里程。 - 当日无有效里程时，日里程补 0，累计总里程、来源协议、dataTime 和 updatedAt 沿用此前最近的有效统计；updatedAt 仍为上一统计周期的计算时间。 + protocolPriority 传入时，逐车按数组顺序选择截至查询日已有有效累计读数的第一个协议；高优先级协议缺报时沿用其历史累计值，不切换累计基准，未列出的协议完全禁用且不会兜底；省略时保持平台默认选源行为。 + NORMAL 结果同时包含日里程、日末累计总里程、实际来源协议、源数据时间和投影更新时间。日里程按同来源相邻自然日累计读数之差计算，缺报期间增量计入恢复上报日；GPS 估算不参与本接口。首次基线、来源切换、累计回退返回 DATA_ANOMALY、dailyMileageKm=null 和 dataQuality。 + 当日缺报时，日里程补 0，累计总里程、来源协议、dataTime 和 updatedAt 沿用同协议历史读数，dataQuality=CARRIED_FORWARD。 operationId: queryDailyMileage security: - AppKeyAuth: [] @@ -141,7 +141,7 @@ paths: 首次请求返回 snapshotId 和 nextCursor；后续请求保持原参数并传回 nextCursor。 快照仅固化授权车辆清单，逐页读取已建立索引的日统计投影，不扫描原始时序明细。 protocolPriority 对区间内每辆车、每个自然日独立生效；未列出的协议完全禁用。 - 某日无有效里程时，dailyMileageKm 补 0，其余里程证据沿用此前最近的有效统计。若终端累计里程回退，返回 DATA_ANOMALY 与 dataQuality=TOTAL_MILEAGE_ROLLBACK，绝不沿用历史值伪装为正常数据。日里程与日末累计总里程独立统计：GPS 轨迹估算仅用于日里程，累计总里程读取每日统计字段 day_end_total_mileage_km。 + 某日无有效里程时，dailyMileageKm 补 0，其余里程证据沿用此前最近的有效统计。若终端累计里程回退，返回 DATA_ANOMALY 与 dataQuality=TOTAL_MILEAGE_ROLLBACK。日里程与累计值使用同一终端来源；日里程为相邻自然日累计差，GPS 估算不参与本接口。单日、区间和分页共用相同计算规则。 operationId: queryDailyMileageRange security: - AppKeyAuth: [] @@ -784,7 +784,7 @@ components: type: number format: double nullable: true - description: 单日里程，km；当日无记录但存在历史累计里程时为 0 + description: 相邻自然日同来源累计值之差，km；跨缺报期增量计入恢复日；缺报日为 0；首次基线、来源变化和累计异常时为 null totalMileageKm: type: number format: double @@ -794,7 +794,7 @@ components: type: string format: date-time nullable: true - description: 当日统计所选来源的最早 first_event_time，跨日基线可早于当日零点，不是自然日起始；历史结转补零或无数据时为 null + description: 日里程起点累计读数的时间；可跨越多个缺报日；历史结转补零或无数据时为 null statisticsEndTime: type: string format: date-time @@ -818,8 +818,8 @@ components: dataQuality: type: string nullable: true - enum: [TOTAL_MILEAGE_ROLLBACK] - description: status=DATA_ANOMALY 时的异常原因；累计里程回退时为 TOTAL_MILEAGE_ROLLBACK + enum: [TOTAL_MILEAGE_ROLLBACK, ODOMETER_SOURCE_CHANGED, NO_PREVIOUS_BASELINE, PREVIOUS_ODOMETER_ANOMALY, CARRIED_FORWARD, outside_daily_range, INVALID_DELTA] + description: CARRIED_FORWARD 表示缺报结转；其他值表示无法连续对账的原因，此时 dailyMileageKm 为 null，保留累计读数及来源证据 status: $ref: '#/components/schemas/DataStatus' MileageRangeResult: @@ -862,8 +862,8 @@ components: dataQuality: type: string nullable: true - enum: [TOTAL_MILEAGE_ROLLBACK] - description: status=DATA_ANOMALY 时的异常原因；累计里程回退时为 TOTAL_MILEAGE_ROLLBACK + enum: [TOTAL_MILEAGE_ROLLBACK, ODOMETER_SOURCE_CHANGED, NO_PREVIOUS_BASELINE, PREVIOUS_ODOMETER_ANOMALY, CARRIED_FORWARD, outside_daily_range, INVALID_DELTA] + description: CARRIED_FORWARD 表示缺报结转；其他值表示无法连续对账的原因，此时 dailyMileageKm 为 null，保留累计读数及来源证据 status: $ref: '#/components/schemas/DataStatus' DataStatus: diff --git a/vehicle-data-platform/apps/api/internal/openplatform/mileage_reconciliation.go b/vehicle-data-platform/apps/api/internal/openplatform/mileage_reconciliation.go new file mode 100644 index 00000000..d34bf524 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/openplatform/mileage_reconciliation.go @@ -0,0 +1,171 @@ +package openplatform + +import ( + "context" + "fmt" + "math" + "sort" + "strings" + "time" +) + +// ReconciledMileageRange uses terminal odometers, including one preceding +// observation per protocol. The same function serves single days and every +// range page: neither the requested start date nor page size changes a result. +func (r *MySQLRepository) ReconciledMileageRange(ctx context.Context, vins []string, start, end string, protocols []string) (map[string]DailyMileage, error) { + if len(vins) == 0 { + return map[string]DailyMileage{}, nil + } + if len(protocols) == 0 { + protocols = []string{"GB32960", "YUTONG_MQTT", "JT808"} + } + placeholders := func(n int) string { return strings.TrimRight(strings.Repeat("?,", n), ",") } + // Unknown-protocol legacy imports and GPS distance are not terminal odometers. + eligible := func(alias string) string { + return fmt.Sprintf(`%[1]s.quality_status IN ('OK','INVALID_DELTA') + AND %[1]s.latest_total_mileage_km > 0 + AND COALESCE(%[1]s.quality_reason,'') <> 'gps_coordinate_accumulation' + AND %[1]s.source_ip NOT IN ('legacy-mysql.lingniu-prod','manual-lingniu-prod-day-mileage') + AND %[1]s.latest_event_time IS NOT NULL + AND %[1]s.latest_event_time < TIMESTAMP(%[1]s.stat_date)+INTERVAL 1 DAY`, alias) + } + filter := `vin IN (` + placeholders(len(vins)) + `) AND protocol IN (` + placeholders(len(protocols)) + `)` + query := `WITH wanted AS ( + SELECT DISTINCT vin,protocol,stat_date FROM vehicle_daily_mileage_source p + WHERE ` + filter + ` AND stat_date BETWEEN ? AND ? AND ` + eligible("p") + ` + UNION ALL + SELECT vin,protocol,MAX(stat_date) AS stat_date FROM vehicle_daily_mileage_source p + WHERE ` + filter + ` AND stat_date 0 { - carried, err = s.repository.LatestMileageBefore(ctx, missingVINs, date, protocols) - if err != nil { - _ = s.repository.Audit(ctx, app.ID, "mileage_query", "error", traceID, len(plates), err.Error()) - return nil, err - } - } results := make([]MileageResult, 0, len(plates)) for _, plate := range plates { vehicle := vehicles[plate] item := MileageResult{VIN: vehicle.VIN, PlateNumber: plate, Date: date, Status: StatusNoData} - if value, ok := values[vehicle.VIN]; ok && validDailyMileage(value) { + if value, ok := values[dailyMileageKey(vehicle.VIN, date)]; ok { fillMileageResult(&item, value, value.MileageKm) - } else if rollbacks[dailyMileageKey(vehicle.VIN, date)] { - fillMileageAnomaly(&item) - } else if value, ok := carried[vehicle.VIN]; ok && validDailyMileage(value) { - fillMileageResult(&item, value, 0) } results = append(results, item) } @@ -556,35 +539,17 @@ func (s *Service) QueryMileageRange(ctx context.Context, appKey, traceID string, vinSet[vehicle.VIN] = struct{}{} } values := map[string]DailyMileage{} - carried := map[string]DailyMileage{} - rollbacks := map[string]bool{} if len(positions) > 0 { vins := make([]string, 0, len(vinSet)) for vin := range vinSet { vins = append(vins, vin) } sort.Strings(vins) - values, err = s.repository.DailyMileageRange(ctx, vins, queryStart.Format("2006-01-02"), queryEnd.Format("2006-01-02"), protocols) + values, err = s.repository.ReconciledMileageRange(ctx, vins, queryStart.Format("2006-01-02"), queryEnd.Format("2006-01-02"), protocols) if err != nil { _ = s.repository.Audit(ctx, app.ID, "mileage_range_query", "error", traceID, snapshot.VehicleCount, err.Error()) return MileageRangeResponse{}, err } - rollbacks, err = s.repository.MileageRollbacks(ctx, vins, queryStart.Format("2006-01-02"), queryEnd.Format("2006-01-02"), protocols) - if err != nil { - _ = s.repository.Audit(ctx, app.ID, "mileage_range_query", "error", traceID, snapshot.VehicleCount, err.Error()) - return MileageRangeResponse{}, err - } - missingVINs := missingMileageRangeInitialVINs(positions, values, rollbacks) - if len(missingVINs) > 0 { - carried, err = s.repository.LatestMileageBefore(ctx, missingVINs, queryStart.Format("2006-01-02"), protocols) - if err != nil { - _ = s.repository.Audit(ctx, app.ID, "mileage_range_query", "error", traceID, snapshot.VehicleCount, err.Error()) - return MileageRangeResponse{}, err - } - if carried == nil { - carried = map[string]DailyMileage{} - } - } } results := make([]MileageRangeResult, 0, len(positions)) for _, position := range positions { @@ -594,14 +559,8 @@ func (s *Service) QueryMileageRange(ctx context.Context, appKey, traceID string, Date: position.date, Status: StatusNoData, } - if value, ok := values[dailyMileageKey(position.vehicle.VIN, position.date)]; ok && validDailyMileage(value) { + if value, ok := values[dailyMileageKey(position.vehicle.VIN, position.date)]; ok { fillMileageRangeResult(&item, value, value.MileageKm) - carried[position.vehicle.VIN] = value - } else if rollbacks[dailyMileageKey(position.vehicle.VIN, position.date)] { - fillMileageRangeAnomaly(&item) - delete(carried, position.vehicle.VIN) - } else if value, ok := carried[position.vehicle.VIN]; ok && validDailyMileage(value) { - fillMileageRangeResult(&item, value, 0) } results = append(results, item) } @@ -1001,6 +960,10 @@ func missingMileageRangeInitialVINs(positions []mileageRangePosition, values map } func fillMileageResult(item *MileageResult, value DailyMileage, dailyMileage float64) { + if value.DataQuality != "" && value.DataTime == "" { + fillMileageAnomaly(item) + return + } if value.Date == item.Date { item.StatisticsStartTime, item.StatisticsEndTime = validatedStatisticsInterval(value.StatisticsStartTime, value.DataTime) } @@ -1014,9 +977,20 @@ func fillMileageResult(item *MileageResult, value DailyMileage, dailyMileage flo sourceProtocol := externalMileageProtocol(value.Protocol) item.SourceProtocol = &sourceProtocol item.Status = StatusNormal + if value.DataQuality != "" { + item.DataQuality = stringPointer(value.DataQuality) + if value.DataQuality != "CARRIED_FORWARD" { + item.Status = StatusDataAnomaly + item.DailyMileageKm = nil + } + } } func fillMileageRangeResult(item *MileageRangeResult, value DailyMileage, dailyMileage float64) { + if value.DataQuality != "" && value.DataTime == "" { + fillMileageRangeAnomaly(item) + return + } item.DailyMileageKm = &dailyMileage totalMileage := value.TotalMileageKm item.TotalMileageKm = &totalMileage @@ -1027,6 +1001,13 @@ func fillMileageRangeResult(item *MileageRangeResult, value DailyMileage, dailyM sourceProtocol := externalMileageProtocol(value.Protocol) item.SourceProtocol = &sourceProtocol item.Status = StatusNormal + if value.DataQuality != "" { + item.DataQuality = stringPointer(value.DataQuality) + if value.DataQuality != "CARRIED_FORWARD" { + item.Status = StatusDataAnomaly + item.DailyMileageKm = nil + } + } } const mileageTotalRollbackQuality = "TOTAL_MILEAGE_ROLLBACK" diff --git a/vehicle-data-platform/apps/api/internal/openplatform/service_test.go b/vehicle-data-platform/apps/api/internal/openplatform/service_test.go index 0ac42b6d..0214a3c5 100644 --- a/vehicle-data-platform/apps/api/internal/openplatform/service_test.go +++ b/vehicle-data-platform/apps/api/internal/openplatform/service_test.go @@ -47,6 +47,56 @@ func (f *fakeRepository) AuthorizedVehicles(_ context.Context, _ uint64, plates f.requestedPlates = append([]string(nil), plates...) return f.vehicles, nil } + +// Repository stub retains fixture values; reconciliation itself is tested with +// actual dated observations in mileage_reconciliation_test.go. +func (f *fakeRepository) ReconciledMileageRange(ctx context.Context, vins []string, start, end string, protocols []string) (map[string]DailyMileage, error) { + f.dailyVINs = append([]string(nil), vins...) + f.rangeVINs = append([]string(nil), vins...) + f.dailyProtocols = protocols + out := map[string]DailyMileage{} + carried := map[string]DailyMileage{} + var missing []string + for _, vin := range vins { + v, ok := f.mileage[dailyMileageKey(vin, start)] + if !ok { + v, ok = f.mileage[vin] + } + if (!ok || !validDailyMileage(v)) && !f.rollbacks[dailyMileageKey(vin, start)] { + missing = append(missing, vin) + } + } + if len(missing) > 0 { + prior, _ := f.LatestMileageBefore(ctx, missing, start, protocols) + for vin, v := range prior { + carried[vin] = v + } + } + first, _ := time.Parse("2006-01-02", start) + last, _ := time.Parse("2006-01-02", end) + for day := first; !day.After(last); day = day.AddDate(0, 0, 1) { + date := day.Format("2006-01-02") + for _, vin := range vins { + key := dailyMileageKey(vin, date) + v, ok := f.mileage[key] + if !ok { + v, ok = f.mileage[vin] + } + if ok && validDailyMileage(v) { + out[key] = v + carried[vin] = v + } else if f.rollbacks[key] { + out[key] = DailyMileage{DataQuality: mileageTotalRollbackQuality} + delete(carried, vin) + } else if v, ok = carried[vin]; ok && validDailyMileage(v) { + v.MileageKm = 0 + out[key] = v + } + } + } + return out, nil +} + func (f *fakeRepository) DailyHydrogen(_ context.Context, vins []string, _ string) (map[string]DailyHydrogen, error) { f.dailyVINs = append([]string(nil), vins...) return f.hydrogen, nil diff --git a/vehicle-data-platform/apps/api/internal/platform/alert.go b/vehicle-data-platform/apps/api/internal/platform/alert.go index a01111ac..5fb29e15 100644 --- a/vehicle-data-platform/apps/api/internal/platform/alert.go +++ b/vehicle-data-platform/apps/api/internal/platform/alert.go @@ -413,6 +413,9 @@ func (s *Service) ActOnAlert(ctx context.Context, id string, request AlertAction func normalizeVehicleEvent(event AlertEvent) AlertEvent { event.EventCategory, event.EventType = canonicalVehicleEvent(event.TriggerType, event.Metric, event.Operator) + if event.RuleID == nativeAlarmRuleID { + event.EventCategory, event.EventType = "safety", "vehicle.safety.gb32960_alarm" + } switch strings.ToLower(strings.TrimSpace(event.Status)) { case "processing": event.ExecutionState = "processing" diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_store.go b/vehicle-data-platform/apps/api/internal/platform/alert_store.go index 6ce57823..c38e8914 100644 --- a/vehicle-data-platform/apps/api/internal/platform/alert_store.go +++ b/vehicle-data-platform/apps/api/internal/platform/alert_store.go @@ -169,6 +169,20 @@ func (s *ProductionStore) AlertEvent(ctx context.Context, id string) (AlertEvent if err != nil { return AlertEvent{}, err } + if event.RuleID == nativeAlarmRuleID { + var payload []byte + if err := s.db.QueryRowContext(ctx, `SELECT fields_json FROM vehicle_native_alarm_evidence WHERE event_id=?`, id).Scan(&payload); err != nil { + return AlertEvent{}, err + } + if err := json.Unmarshal(payload, &event.NativeAlarmFields); err != nil { + return AlertEvent{}, err + } + if description, valid := DescribeNativeAlarm(event.NativeAlarmFields); valid { + event.RuleName = description.Title + event.NativeAlarmNames = description.Names + event.NativeAlarmReservedBits = description.ReservedBits + } + } rows, err := s.db.QueryContext(ctx, alertActionSelect, id) if err != nil { return AlertEvent{}, err diff --git a/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go b/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go index 5dbe9f92..17ebd9d6 100644 --- a/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go +++ b/vehicle-data-platform/apps/api/internal/platform/alert_stream_evaluator.go @@ -22,6 +22,9 @@ func evaluateAlertStreamRecordsTx(ctx context.Context, tx *sql.Tx, records []Ale if err != nil { return result, err } + if err := recordNativeAlarmsTx(ctx, tx, records, &result); err != nil { + return result, err + } if len(rules) == 0 { return result, nil } diff --git a/vehicle-data-platform/apps/api/internal/platform/corrected_hydrogen_test.go b/vehicle-data-platform/apps/api/internal/platform/corrected_hydrogen_test.go new file mode 100644 index 00000000..7ecfbbbb --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/corrected_hydrogen_test.go @@ -0,0 +1,51 @@ +package platform + +import ( + "context" + "net/url" + "testing" +) + +func TestCorrectedDailyHydrogenRate(t *testing.T) { + physical, corrected, mixed, zero := 10.0, 12.0, 200.0, 0.0 + row := DailyMileageRow{HydrogenConsumptionKg: &physical, HydrogenSOCBalancedKg: &corrected, PureHydrogenMileageKm: 100, MixedMileageKm: &mixed} + if got := correctedDailyHydrogenRate(row); got == nil || *got != 6 { + t.Fatalf("rate must use corrected mass and matching mileage: %v", got) + } + row.MixedMileageKm = nil + if got := correctedDailyHydrogenRate(row); got == nil || *got != 12 { + t.Fatalf("fallback mileage: %v", got) + } + row.HydrogenSOCBalancedKg = nil + if got := correctedDailyHydrogenRate(row); got != nil { + t.Fatalf("missing correction must not use physical mass: %v", *got) + } + row.HydrogenSOCBalancedKg = &zero + if got := correctedDailyHydrogenRate(row); got == nil || *got != 0 { + t.Fatalf("real zero must survive: %v", got) + } + row.PureHydrogenMileageKm = 0 + if got := correctedDailyHydrogenRate(row); got != nil { + t.Fatalf("zero mileage must have no rate: %v", *got) + } +} + +type distinctHydrogenStore struct{ *MockStore } + +func (s *distinctHydrogenStore) DailyMileage(context.Context, url.Values) (Page[DailyMileageRow], error) { + physical, corrected, distance := 12.813, 8.714, 77.0 + return Page[DailyMileageRow]{Items: []DailyMileageRow{{VIN: "LA9GG64L1NBAF4167", Date: "2026-09-01", HydrogenConsumptionKg: &physical, HydrogenSOCBalancedKg: &corrected, MixedMileageKm: &distance}}}, nil +} +func TestDailyMileageHistoricalPublicFieldsUseCorrection(t *testing.T) { + result, err := NewService(&distinctHydrogenStore{NewMockStore()}).DailyMileage(context.Background(), url.Values{"dateFrom": {"2026-09-01"}, "dateTo": {"2026-09-01"}}) + if err != nil { + t.Fatal(err) + } + row := result.Items[0] + if row.HydrogenConsumptionKg == nil || *row.HydrogenConsumptionKg != 8.714 || row.HydrogenPhysicalConsumptionKg == nil || *row.HydrogenPhysicalConsumptionKg != 12.813 { + t.Fatalf("historical primary amount must be corrected, with physical amount separately available: %+v", row) + } + if row.HydrogenConsumptionKgPer100Km == nil || *row.HydrogenConsumptionKgPer100Km != 8.714*100/77 { + t.Fatalf("historical rate mismatch: %+v", row) + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/hydrogen_evidence_test.go b/vehicle-data-platform/apps/api/internal/platform/hydrogen_evidence_test.go index 9cae489c..4f2642d4 100644 --- a/vehicle-data-platform/apps/api/internal/platform/hydrogen_evidence_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/hydrogen_evidence_test.go @@ -34,6 +34,9 @@ func TestProductionStoreHydrogenDailyEvidenceReturnsParametersIntervalsAndRawEve if result.Parameters.BatteryCapacityKWh != 21.04 || result.RefuelAmountKg == nil || *result.RefuelAmountKg != 8.976 || result.ChargeEnergyKWh == nil || *result.ChargeEnergyKWh != 12.5 || len(result.Intervals) != 1 || result.Intervals[0].StartEventID != "event-start" || result.Intervals[0].EndEventID != "event-end" { t.Fatalf("result=%#v", result) } + if result.ConsumptionKgPer100Km == nil || *result.ConsumptionKgPer100Km != 5.562 || result.PhysicalConsumptionKgPer100Km == nil || *result.PhysicalConsumptionKgPer100Km != 5.516 { + t.Fatalf("evidence must expose corrected rate separately from physical rate: %+v", result) + } if err := mock.ExpectationsWereMet(); err != nil { t.Fatal(err) } diff --git a/vehicle-data-platform/apps/api/internal/platform/model.go b/vehicle-data-platform/apps/api/internal/platform/model.go index df7001be..bd7e2e79 100644 --- a/vehicle-data-platform/apps/api/internal/platform/model.go +++ b/vehicle-data-platform/apps/api/internal/platform/model.go @@ -1,6 +1,9 @@ package platform -import "time" +import ( + "encoding/json" + "time" +) type Page[T any] struct { Items []T `json:"items"` @@ -1000,37 +1003,40 @@ type AlertRuleLifecycleRequest struct { } type AlertEvent struct { - ID string `json:"id"` - EventType string `json:"eventType"` - EventCategory string `json:"eventCategory"` - ExecutionState string `json:"executionState"` - RuleID string `json:"ruleId"` - RuleName string `json:"ruleName"` - RuleVersion int `json:"ruleVersion"` - Severity string `json:"severity"` - TriggerType string `json:"triggerType"` - Status string `json:"status"` - VIN string `json:"vin"` - Plate string `json:"plate"` - Protocol string `json:"protocol"` - Metric string `json:"metric"` - Operator string `json:"operator"` - TriggerValue float64 `json:"triggerValue"` - Threshold float64 `json:"threshold"` - ThresholdHigh float64 `json:"thresholdHigh"` - Unit string `json:"unit"` - DurationSec int `json:"durationSec"` - Location string `json:"location"` - Longitude *float64 `json:"longitude,omitempty"` - Latitude *float64 `json:"latitude,omitempty"` - SourceEventID string `json:"sourceEventId"` - EventAt string `json:"eventAt"` - ReceivedAt string `json:"receivedAt"` - TriggeredAt string `json:"triggeredAt"` - RecoveredAt string `json:"recoveredAt"` - Handler string `json:"handler"` - Version int `json:"version"` - Actions []AlertAction `json:"actions,omitempty"` + NativeAlarmNames []string `json:"nativeAlarmNames,omitempty"` + NativeAlarmReservedBits []int `json:"nativeAlarmReservedBits,omitempty"` + NativeAlarmFields map[string]json.RawMessage `json:"nativeAlarmFields,omitempty"` + ID string `json:"id"` + EventType string `json:"eventType"` + EventCategory string `json:"eventCategory"` + ExecutionState string `json:"executionState"` + RuleID string `json:"ruleId"` + RuleName string `json:"ruleName"` + RuleVersion int `json:"ruleVersion"` + Severity string `json:"severity"` + TriggerType string `json:"triggerType"` + Status string `json:"status"` + VIN string `json:"vin"` + Plate string `json:"plate"` + Protocol string `json:"protocol"` + Metric string `json:"metric"` + Operator string `json:"operator"` + TriggerValue float64 `json:"triggerValue"` + Threshold float64 `json:"threshold"` + ThresholdHigh float64 `json:"thresholdHigh"` + Unit string `json:"unit"` + DurationSec int `json:"durationSec"` + Location string `json:"location"` + Longitude *float64 `json:"longitude,omitempty"` + Latitude *float64 `json:"latitude,omitempty"` + SourceEventID string `json:"sourceEventId"` + EventAt string `json:"eventAt"` + ReceivedAt string `json:"receivedAt"` + TriggeredAt string `json:"triggeredAt"` + RecoveredAt string `json:"recoveredAt"` + Handler string `json:"handler"` + Version int `json:"version"` + Actions []AlertAction `json:"actions,omitempty"` } type AlertAction struct { @@ -1874,6 +1880,12 @@ type LatestTelemetryResponse struct { } type DailyMileageRow struct { + Province string `json:"province,omitempty"` + City string `json:"city,omitempty"` + Region string `json:"region,omitempty"` + LocationTime string `json:"locationTime,omitempty"` + LocationStatus string `json:"locationStatus"` + HydrogenPhysicalConsumptionKg *float64 `json:"hydrogenPhysicalConsumptionKg,omitempty"` VIN string `json:"vin"` Plate string `json:"plate"` Date string `json:"date"` @@ -1942,32 +1954,33 @@ type HydrogenIntervalEvidenceRow struct { } type HydrogenDailyEvidence struct { - VIN string `json:"vin"` - Plate string `json:"plate"` - Date string `json:"date"` - Source string `json:"source"` - RawConsumptionKg float64 `json:"rawConsumptionKg"` - BatterySOCDeltaPct *float64 `json:"batterySocDeltaPct,omitempty"` - BatteryDischargeKWh *float64 `json:"batteryDischargeKWh,omitempty"` - BatteryEquivalentKg *float64 `json:"batteryEquivalentKg,omitempty"` - SOCBalancedConsumptionKg *float64 `json:"socBalancedConsumptionKg,omitempty"` - MixedMileageKm float64 `json:"mixedMileageKm"` - PureElectricMileageKm float64 `json:"pureElectricMileageKm"` - ConsumptionKgPer100Km *float64 `json:"consumptionKgPer100Km,omitempty"` - SOCBalancedKgPer100Km *float64 `json:"socBalancedKgPer100Km,omitempty"` - SampleCount int `json:"sampleCount"` - RefuelCount int `json:"refuelCount"` - RefuelAmountKg *float64 `json:"refuelAmountKg,omitempty"` - ChargeCount int `json:"chargeCount"` - ChargeEnergyKWh *float64 `json:"chargeEnergyKWh,omitempty"` - ValidSegmentCount int `json:"validSegmentCount"` - InvalidSegmentCount int `json:"invalidSegmentCount"` - QualityStatus string `json:"qualityStatus"` - QualityReason string `json:"qualityReason"` - AlgorithmVersion string `json:"algorithmVersion"` - Parameters HydrogenCalculationParameterEvidence `json:"parameters"` - Intervals []HydrogenIntervalEvidenceRow `json:"intervals"` - CalculatedAt string `json:"calculatedAt"` + PhysicalConsumptionKgPer100Km *float64 `json:"physicalConsumptionKgPer100Km,omitempty"` + VIN string `json:"vin"` + Plate string `json:"plate"` + Date string `json:"date"` + Source string `json:"source"` + RawConsumptionKg float64 `json:"rawConsumptionKg"` + BatterySOCDeltaPct *float64 `json:"batterySocDeltaPct,omitempty"` + BatteryDischargeKWh *float64 `json:"batteryDischargeKWh,omitempty"` + BatteryEquivalentKg *float64 `json:"batteryEquivalentKg,omitempty"` + SOCBalancedConsumptionKg *float64 `json:"socBalancedConsumptionKg,omitempty"` + MixedMileageKm float64 `json:"mixedMileageKm"` + PureElectricMileageKm float64 `json:"pureElectricMileageKm"` + ConsumptionKgPer100Km *float64 `json:"consumptionKgPer100Km,omitempty"` + SOCBalancedKgPer100Km *float64 `json:"socBalancedKgPer100Km,omitempty"` + SampleCount int `json:"sampleCount"` + RefuelCount int `json:"refuelCount"` + RefuelAmountKg *float64 `json:"refuelAmountKg,omitempty"` + ChargeCount int `json:"chargeCount"` + ChargeEnergyKWh *float64 `json:"chargeEnergyKWh,omitempty"` + ValidSegmentCount int `json:"validSegmentCount"` + InvalidSegmentCount int `json:"invalidSegmentCount"` + QualityStatus string `json:"qualityStatus"` + QualityReason string `json:"qualityReason"` + AlgorithmVersion string `json:"algorithmVersion"` + Parameters HydrogenCalculationParameterEvidence `json:"parameters"` + Intervals []HydrogenIntervalEvidenceRow `json:"intervals"` + CalculatedAt string `json:"calculatedAt"` } // MileageQuery is the POST contract used by mileage statistics and daily diff --git a/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go b/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go index fbfb2329..b8dc10df 100644 --- a/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go +++ b/vehicle-data-platform/apps/api/internal/platform/mysql_queries.go @@ -565,7 +565,8 @@ func buildDailyMileageSQL(query url.Values) SQLQuery { LEFT JOIN vehicle_open_daily_energy h ON h.vin COLLATE utf8mb4_unicode_ci = m.vin COLLATE utf8mb4_unicode_ci AND h.stat_date = m.stat_date - AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT')` + AND h.energy_type = 'HYDROGEN' AND h.quality_status IN ('OK','SUSPECT') +LEFT JOIN vehicle_daily_geography g ON g.vin = m.vin COLLATE utf8mb4_unicode_ci AND g.stat_date = m.stat_date` if query.Get("deduplicate") == "1" || strings.EqualFold(query.Get("deduplicate"), "true") { selectionOrder := `m.daily_mileage_km DESC, m.protocol ASC` @@ -594,7 +595,7 @@ LEFT JOIN vehicle_open_daily_energy h `h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, ` + `CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END AS hydrogen_evidence_available, ` + `COALESCE(h.quality_status, '') AS hydrogen_quality_status, COALESCE(h.quality_reason, '') AS hydrogen_quality_reason, ` + - `COALESCE(h.algorithm_version, '') AS hydrogen_algorithm_version, m.protocol ` + + `COALESCE(h.algorithm_version, '') AS hydrogen_algorithm_version, m.protocol, ` + dailyGeographyProjectionSQL + ` ` + `FROM (` + pageSQL + `) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC`, Args: args, CountText: `SELECT COUNT(*) FROM (SELECT m.vin ` + groupSQL + `) vehicle_daily_mileage_count`, @@ -613,7 +614,7 @@ LEFT JOIN vehicle_open_daily_energy h `COALESCE(m.pure_hydrogen_mileage_km, 0), h.consumption_kg, h.consumption_kg_per_100km, h.soc_balanced_consumption_kg, ` + `h.soc_balanced_kg_per_100km, h.pure_electric_mileage_km, h.mixed_mileage_km, h.battery_soc_delta_pct, ` + `h.charge_count, h.charge_energy_kwh, h.refuel_count, h.refuel_amount_kg, CASE WHEN h.evidence_json IS NOT NULL THEN 1 ELSE 0 END, ` + - `COALESCE(h.quality_status, ''), COALESCE(h.quality_reason, ''), COALESCE(h.algorithm_version, ''), m.protocol ` + + `COALESCE(h.quality_status, ''), COALESCE(h.quality_reason, ''), COALESCE(h.algorithm_version, ''), m.protocol, ` + dailyGeographyProjectionSQL + ` ` + `FROM (SELECT m.* ` + filterFromSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC LIMIT ? OFFSET ?) m` + enrichmentSQL + ` ORDER BY m.stat_date DESC, m.vin ASC, m.protocol ASC`, Args: args, CountText: `SELECT COUNT(*) ` + filterFromSQL, @@ -710,7 +711,7 @@ func buildMileageStatisticsBaseSQL(query url.Values) (string, []any) { dailyMileageExpression + ` AS daily_mileage_km, ` + pureHydrogenMileageExpression + ` AS pure_hydrogen_mileage_km, ` + `CASE WHEN MAX(COALESCE(h.mixed_mileage_km,0))>0 THEN MAX(h.mixed_mileage_km) ELSE ` + pureHydrogenMileageExpression + ` END AS hydrogen_matched_mileage_km, ` + - `MAX(h.consumption_kg) AS hydrogen_consumption_kg, ` + + `MAX(h.soc_balanced_consumption_kg) AS hydrogen_consumption_kg, ` + latestMileageExpression + ` AS latest_mileage_km ` + `FROM vehicle_daily_mileage m LEFT JOIN vehicle_identity_binding b ON b.vin = m.vin @@ -917,3 +918,5 @@ func mustInt(value string) int { n, _ := strconv.Atoi(value) return n } + +const dailyGeographyProjectionSQL = `COALESCE(g.province,''),COALESCE(g.city,''),COALESCE(g.region,''),COALESCE(DATE_FORMAT(g.location_time,'%Y-%m-%dT%H:%i:%s.%f+08:00'),''),CASE COALESCE(g.status,'PENDING') WHEN 'RESOLVED' THEN '已解析' WHEN 'NO_LOCATION' THEN '当天无有效定位' WHEN 'ERROR' THEN '解析失败，后台重试中' ELSE '后台待解析' END` diff --git a/vehicle-data-platform/apps/api/internal/platform/native_alarm.go b/vehicle-data-platform/apps/api/internal/platform/native_alarm.go new file mode 100644 index 00000000..1af50059 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/native_alarm.go @@ -0,0 +1,142 @@ +package platform + +import ( + "context" + "database/sql" + "encoding/json" + "math" + "time" +) + +const nativeAlarmRuleID = "native-gb32960-alarm" + +// Require the complete 0x07 unit: absent/invalid data must never clear an alarm. +func nativeAlarmFields(record AlertStreamRecord) (map[string]json.RawMessage, int, bool, bool) { + if record.Protocol != "GB32960" { + return nil, 0, false, false + } + fields := make(map[string]json.RawMessage, 6) + level, levelOK := alertStreamRawNumber(record.Fields["gb32960.alarm.max_alarm_level"]) + flag, flagOK := alertStreamRawNumber(record.Fields["gb32960.alarm.general_alarm_flag"]) + if !levelOK || !flagOK || level < 0 || level > 3 || math.Trunc(level) != level || flag < 0 || flag > math.MaxUint32 || math.Trunc(flag) != flag { + return nil, 0, false, false + } + mask := nativeAlarm2016Mask + if nativeAlarmVersion(record.Fields) == "V2025" { + mask = math.MaxUint32 + } + active := level > 0 || uint32(flag)&mask != 0 + if version := record.Fields["gb32960.header.version"]; len(version) > 0 { + fields["gb32960.header.version"] = version + } + for _, name := range []string{"max_alarm_level", "general_alarm_flag", "battery_faults", "motor_faults", "engine_faults", "other_faults"} { + key := "gb32960.alarm." + name + raw := record.Fields[key] + if name != "max_alarm_level" && name != "general_alarm_flag" { + // The gateway's FIELDS contract serializes complex KV values as JSON strings. + var encoded string + if json.Unmarshal(raw, &encoded) == nil { + raw = json.RawMessage(encoded) + } + var codes []string + if len(raw) == 0 || string(raw) == "null" || json.Unmarshal(raw, &codes) != nil || codes == nil { + return nil, 0, false, false + } + active = active || len(codes) > 0 + } + fields[key] = raw + } + return fields, int(level), active, true +} + +// Runs in the same transaction as the Kafka checkpoint, independently of rules. +func recordNativeAlarmsTx(ctx context.Context, tx *sql.Tx, records []AlertStreamRecord, result *AlertEvaluationResult) error { + eligible := make([]AlertStreamRecord, 0) + for _, record := range records { + if _, _, _, ok := nativeAlarmFields(record); ok && !record.Late { + eligible = append(eligible, record) + } + } + if len(eligible) == 0 { + return nil + } + metadata, err := loadAlertStreamVehicleMetadata(ctx, tx, eligible) + if err != nil { + return err + } + for _, record := range eligible { + fields, level, active, _ := nativeAlarmFields(record) + // Serialize concurrent consumers for this VIN, including the first observation. + if _, err = tx.ExecContext(ctx, `INSERT IGNORE INTO vehicle_native_alarm_state(vin,last_event_at) VALUES(?,'1970-01-01 00:00:00')`, record.VIN); err != nil { + return err + } + var lastAt time.Time + var eventID string + if err = tx.QueryRowContext(ctx, `SELECT last_event_at,active_event_id FROM vehicle_native_alarm_state WHERE vin=? FOR UPDATE`, record.VIN).Scan(&lastAt, &eventID); err != nil { + return err + } + if !record.EventAt.After(lastAt) { + result.LateObservations++ + continue + } + if active && eventID == "" { + eventID, err = newAlertID("alert") + if err != nil { + return err + } + severity := "minor" + if level == 2 { + severity = "major" + } + if level == 3 { + severity = "critical" + } + description, _ := DescribeNativeAlarm(fields) + title := description.Title + rule := AlertRule{ID: nativeAlarmRuleID, Name: title, Severity: severity, TriggerType: "metric", Metric: "alarm_active", Operator: "eq", Threshold: 1} + item := alertStreamEvidence(record, metadata[record.VIN], nil) + shanghai := time.FixedZone("Asia/Shanghai", 8*60*60) + item.EventAt = record.EventAt.In(shanghai).Format("2006-01-02 15:04:05.999") + item.ReceivedAt = record.ReceivedAt.In(shanghai).Format("2006-01-02 15:04:05.999") + query, args := buildAlertEventInsert(eventID, nativeAlarmRuleID+"|"+record.VIN+"|GB32960", rule, item, 1) + // Missing coordinates must not appear as a real position at (0,0). + if !item.HasLocation { + args[17], args[18], args[19] = "", nil, nil + } + if _, err = tx.ExecContext(ctx, query, args...); err != nil { + return err + } + payload, marshalErr := json.Marshal(fields) + if marshalErr != nil { + return marshalErr + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_native_alarm_evidence(event_id,fields_json) VALUES(?,?)`, eventID, string(payload)); err != nil { + return err + } + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'trigger','','unprocessed','native-gb32960','车辆上报原生告警')`, eventID); err != nil { + return err + } + result.Opened++ + } else if !active && eventID != "" { + update, updateErr := tx.ExecContext(ctx, `UPDATE vehicle_alert_event SET status='recovered',recovered_at=?,version=version+1 WHERE id=? AND status IN ('unprocessed','processing')`, record.EventAt, eventID) + if updateErr != nil { + return updateErr + } + count, countErr := update.RowsAffected() + if countErr != nil { + return countErr + } + if count > 0 { + if _, err = tx.ExecContext(ctx, `INSERT INTO vehicle_alert_event_action(event_id,action,from_status,to_status,actor,note) VALUES(?,'recover','','recovered','native-gb32960','车辆上报原生告警解除')`, eventID); err != nil { + return err + } + result.Recovered++ + } + eventID = "" + } + if _, err = tx.ExecContext(ctx, `UPDATE vehicle_native_alarm_state SET last_event_at=?,active_event_id=? WHERE vin=?`, record.EventAt, eventID, record.VIN); err != nil { + return err + } + } + return nil +} diff --git a/vehicle-data-platform/apps/api/internal/platform/native_alarm_description.go b/vehicle-data-platform/apps/api/internal/platform/native_alarm_description.go new file mode 100644 index 00000000..6be05a69 --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/native_alarm_description.go @@ -0,0 +1,84 @@ +package platform + +import ( + "encoding/json" + "fmt" + "strings" +) + +// GB/T 32960.3-2016, table 18. Bits 19–31 are reserved, not faults. +var nativeAlarmBitNames = [...]string{ + "温度差异报警", "电池高温报警", "车载储能装置类型过压报警", "车载储能装置类型欠压报警", + "SOC低报警", "单体电池过压报警", "单体电池欠压报警", "SOC过高报警", "SOC跳变报警", + "可充电储能系统不匹配报警", "电池单体一致性差报警", "绝缘报警", "DC-DC温度报警", "制动系统报警", + "DC-DC状态报警", "驱动电机控制器温度报警", "高压互锁状态报警", "驱动电机温度报警", "车载储能装置类型过充报警", +} + +const nativeAlarm2016Mask uint32 = (1 << 19) - 1 + +// NativeAlarmDescription is also used by the repair command so historical and +// newly ingested events use the same interpretation of their saved evidence. +type NativeAlarmDescription struct { + Title string `json:"title"` + Names []string `json:"names"` + ReservedBits []int `json:"reservedBits"` + Active bool `json:"active"` +} + +func nativeAlarmVersion(fields map[string]json.RawMessage) string { + var version string + _ = json.Unmarshal(fields["gb32960.header.version"], &version) + return version +} + +func DescribeNativeAlarm(fields map[string]json.RawMessage) (NativeAlarmDescription, bool) { + fields, level, active, valid := nativeAlarmFields(AlertStreamRecord{Protocol: "GB32960", Fields: fields}) + result := NativeAlarmDescription{Names: []string{}, ReservedBits: []int{}, Active: active} + if !valid { + return result, false + } + value, _ := alertStreamRawNumber(fields["gb32960.alarm.general_alarm_flag"]) + flag := uint32(value) + for bit, name := range nativeAlarmBitNames { + if flag&(uint32(1)< 0 { + result.Names = append(result.Names, fmt.Sprintf("%d级故障（车辆未上报具体故障项）", level)) + } + if len(result.Names) == 0 { + result.Title = "无标准告警" + return result, true + } + result.Title = strings.Join(result.Names, "、") + // Persist a useful, bounded title in the existing VARCHAR(80) column; detail + // keeps every name and code, including concurrent faults. + if len([]rune(result.Title)) > 80 { + first := []rune(result.Names[0]) + if len(first) > 55 { + first = first[:55] + } + result.Title = fmt.Sprintf("%s 等%d项告警", string(first), len(result.Names)) + } + return result, true +} diff --git a/vehicle-data-platform/apps/api/internal/platform/native_alarm_test.go b/vehicle-data-platform/apps/api/internal/platform/native_alarm_test.go new file mode 100644 index 00000000..41bdc27c --- /dev/null +++ b/vehicle-data-platform/apps/api/internal/platform/native_alarm_test.go @@ -0,0 +1,203 @@ +package platform + +import ( + "encoding/json" + "fmt" + "regexp" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" +) + +func nativeAlarmRecord(level, flag string) AlertStreamRecord { + fields := map[string]json.RawMessage{"gb32960.alarm.max_alarm_level": json.RawMessage(level), "gb32960.alarm.general_alarm_flag": json.RawMessage(flag)} + for _, name := range []string{"battery_faults", "motor_faults", "engine_faults", "other_faults"} { + fields["gb32960.alarm."+name] = json.RawMessage(`[]`) + } + return AlertStreamRecord{Protocol: "GB32960", VIN: "VIN1", SourceEventID: "native-source", EventAt: time.Date(2026, 9, 17, 1, 0, 0, 0, time.UTC), ReceivedAt: time.Date(2026, 9, 17, 1, 0, 1, 0, time.UTC), Fields: fields, Valid: true} +} + +func TestNativeAlarmFields(t *testing.T) { + for _, tc := range []struct { + name, level, flag string + active, valid bool + }{ + {"clear", "0", `"0x00000000"`, false, true}, + {"level", "3", `0`, true, true}, + {"flag", "0", `"0x00000004"`, true, true}, + {"reserved only", "0", `"0x00300000"`, false, true}, + {"reserved and insulation", "0", `"0x00380800"`, true, true}, + {"abnormal", "254", `0`, false, false}, + {"invalid", "255", `0`, false, false}, + {"fraction", "1.5", `0`, false, false}, + {"bad bitmap", "0", `"garbage"`, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, active, valid := nativeAlarmFields(nativeAlarmRecord(tc.level, tc.flag)) + if active != tc.active || valid != tc.valid { + t.Fatalf("active=%v valid=%v", active, valid) + } + }) + } + encodedRecord := nativeAlarmRecord(`"2"`, `"0x00000004"`) + for _, name := range []string{"battery_faults", "motor_faults", "engine_faults", "other_faults"} { + encodedRecord.Fields["gb32960.alarm."+name] = json.RawMessage(`"[]"`) + } + encodedRecord.Fields["gb32960.alarm.motor_faults"] = json.RawMessage(`"[\"0x00000001\"]"`) + fields, _, active, valid := nativeAlarmFields(encodedRecord) + if !valid || !active || string(fields["gb32960.alarm.motor_faults"]) != `["0x00000001"]` { + t.Fatalf("gateway string-encoded fault arrays lost: %v", fields) + } + record := nativeAlarmRecord("0", "0") + record.Fields["gb32960.alarm.motor_faults"] = json.RawMessage(`["0x00000001"]`) + if _, _, active, valid := nativeAlarmFields(record); !active || !valid { + t.Fatal("fault code alone must activate alarm") + } + delete(record.Fields, "gb32960.alarm.battery_faults") + if _, _, _, valid := nativeAlarmFields(record); valid { + t.Fatal("partial unit must not activate or recover") + } +} + +func TestNativeAlarmLifecycleWithoutRules(t *testing.T) { + for _, tc := range []struct { + name, existing string + clear, stale, closed bool + }{ + {name: "open without rules"}, + {name: "continuous alarm deduplicated", existing: "existing"}, + {name: "recover", existing: "existing", clear: true}, + {name: "manual closure preserved", existing: "existing", clear: true, closed: true}, + {name: "out of order ignored", existing: "existing", clear: true, stale: true}, + } { + t.Run(tc.name, func(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + record := nativeAlarmRecord("2", `"0x00000004"`) + if tc.clear { + record = nativeAlarmRecord("0", "0") + } + mock.ExpectBegin() + mock.ExpectQuery(regexp.QuoteMeta(alertRuleSelect + `WHERE enabled=1 AND archived_at IS NULL AND metric<>'freshness_sec' ORDER BY id FOR UPDATE`)).WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectQuery(`SELECT v.vin,`).WithArgs("VIN1").WillReturnRows(sqlmock.NewRows([]string{"vin", "plate", "oem", "model", "company"}).AddRow("VIN1", "粤A12345", "", "", "")) + mock.ExpectExec(`INSERT IGNORE INTO vehicle_native_alarm_state`).WithArgs("VIN1").WillReturnResult(sqlmock.NewResult(1, 1)) + last := record.EventAt.Add(-time.Second) + if tc.stale { + last = record.EventAt.Add(time.Second) + } + mock.ExpectQuery(`SELECT last_event_at,active_event_id`).WithArgs("VIN1").WillReturnRows(sqlmock.NewRows([]string{"last_event_at", "active_event_id"}).AddRow(last, tc.existing)) + if !tc.stale { + if tc.existing == "" { + mock.ExpectExec(regexp.QuoteMeta(alertEventInsertSQL)).WithArgs(sqlmock.AnyArg(), nativeAlarmRuleID+"|VIN1|GB32960", nativeAlarmRuleID, "车载储能装置类型过压报警", 0, "major", "metric", "VIN1", "粤A12345", "GB32960", "alarm_active", "eq", float64(1), float64(1), float64(0), "", 0, "", nil, nil, "native-source", sqlmock.AnyArg(), sqlmock.AnyArg()).WillReturnResult(sqlmock.NewResult(1, 1)) + payload, _ := json.Marshal(record.Fields) + mock.ExpectExec(`INSERT INTO vehicle_native_alarm_evidence`).WithArgs(sqlmock.AnyArg(), string(payload)).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`INSERT INTO vehicle_alert_event_action`).WillReturnResult(sqlmock.NewResult(1, 1)) + } else if tc.clear { + count := int64(1) + if tc.closed { + count = 0 + } + mock.ExpectExec(`UPDATE vehicle_alert_event SET status='recovered'`).WithArgs(record.EventAt, tc.existing).WillReturnResult(sqlmock.NewResult(0, count)) + if !tc.closed { + mock.ExpectExec(`INSERT INTO vehicle_alert_event_action`).WillReturnResult(sqlmock.NewResult(1, 1)) + } + } + mock.ExpectExec(`UPDATE vehicle_native_alarm_state`).WithArgs(record.EventAt, sqlmock.AnyArg(), "VIN1").WillReturnResult(sqlmock.NewResult(0, 1)) + } + mock.ExpectCommit() + tx, err := db.Begin() + if err != nil { + t.Fatal(err) + } + result, err := evaluateAlertStreamRecordsTx(t.Context(), tx, []AlertStreamRecord{record}, record.ReceivedAt) + if err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if result.RulesEvaluated != 0 { + t.Fatal("native alarms require no automation") + } + if tc.existing == "" && result.Opened != 1 { + t.Fatalf("not opened: %+v", result) + } + if tc.existing != "" && result.Opened != 0 { + t.Fatal("duplicate event") + } + if tc.clear && !tc.closed && !tc.stale && result.Recovered != 1 { + t.Fatal("not recovered") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestNativeAlarmLateAndMissingUnitsDoNotWrite(t *testing.T) { + db, mock, _ := sqlmock.New() + defer db.Close() + late := nativeAlarmRecord("3", "1") + late.Late = true + missing := nativeAlarmRecord("0", "0") + delete(missing.Fields, "gb32960.alarm.general_alarm_flag") + mock.ExpectBegin() + mock.ExpectCommit() + tx, _ := db.Begin() + if err := recordNativeAlarmsTx(t.Context(), tx, []AlertStreamRecord{late, missing}, &AlertEvaluationResult{}); err != nil { + t.Fatal(err) + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestNativeAlarmDescriptionDecodesSpecificFaultsAndReservedBits(t *testing.T) { + record := nativeAlarmRecord("0", `"0x00380800"`) + d, ok := DescribeNativeAlarm(record.Fields) + if !ok || !d.Active || d.Title != "绝缘报警" || len(d.Names) != 1 || len(d.ReservedBits) != 3 || d.ReservedBits[0] != 19 { + t.Fatalf("wrong description: %+v", d) + } + record = nativeAlarmRecord("0", `"0x00380000"`) + d, ok = DescribeNativeAlarm(record.Fields) + if !ok || d.Active || len(d.Names) != 0 { + t.Fatalf("reserved bits became faults: %+v", d) + } + record = nativeAlarmRecord("2", `"0x00020810"`) + record.Fields["gb32960.alarm.motor_faults"] = json.RawMessage(`"[\"0x12345678\"]"`) + d, ok = DescribeNativeAlarm(record.Fields) + if !ok || len(d.Names) != 4 || d.Names[0] != "SOC低报警" || d.Names[1] != "绝缘报警" || d.Names[2] != "驱动电机温度报警" || d.Names[3] != "驱动电机故障码 0x12345678（厂商定义）" { + t.Fatalf("lost concurrent faults: %+v", d) + } + record = nativeAlarmRecord("3", `"0x00000000"`) + d, ok = DescribeNativeAlarm(record.Fields) + if !ok || !d.Active || d.Title != "3级故障（车辆未上报具体故障项）" { + t.Fatalf("invented fault meaning: %+v", d) + } + for bit, name := range nativeAlarmBitNames { + record = nativeAlarmRecord("0", fmt.Sprintf("%d", uint32(1)< 80 { + t.Fatalf("title overflow or lost detail: %+v", d) + } +} diff --git a/vehicle-data-platform/apps/api/internal/platform/production_store.go b/vehicle-data-platform/apps/api/internal/platform/production_store.go index 36f2fa0c..71d90459 100644 --- a/vehicle-data-platform/apps/api/internal/platform/production_store.go +++ b/vehicle-data-platform/apps/api/internal/platform/production_store.go @@ -1158,7 +1158,7 @@ func (s *ProductionStore) DailyMileage(ctx context.Context, query url.Values) (P var evidenceAvailable int if err := rows.Scan(&row.VIN, &row.Plate, &row.Date, &row.StartMileageKm, &row.EndMileageKm, &row.DailyMileageKm, &row.PureHydrogenMileageKm, &hydrogen, &physicalRate, &balanced, &balancedRate, &pureElectric, &mixed, &socDelta, &chargeCount, &chargeEnergy, &refuelCount, &refuelAmount, - &evidenceAvailable, &row.HydrogenQualityStatus, &row.HydrogenQualityReason, &row.HydrogenAlgorithmVersion, &row.Source); err != nil { + &evidenceAvailable, &row.HydrogenQualityStatus, &row.HydrogenQualityReason, &row.HydrogenAlgorithmVersion, &row.Source, &row.Province, &row.City, &row.Region, &row.LocationTime, &row.LocationStatus); err != nil { return Page[DailyMileageRow]{}, err } if hydrogen.Valid { @@ -1237,7 +1237,8 @@ LIMIT 1`, vin, date) result.BatteryDischargeKWh = nullableFloatPointer(discharge) result.BatteryEquivalentKg = nullableFloatPointer(equivalent) result.SOCBalancedConsumptionKg = nullableFloatPointer(balanced) - result.ConsumptionKgPer100Km = nullableFloatPointer(physicalRate) + result.PhysicalConsumptionKgPer100Km = nullableFloatPointer(physicalRate) + result.ConsumptionKgPer100Km = nullableFloatPointer(balancedRate) result.SOCBalancedKgPer100Km = nullableFloatPointer(balancedRate) result.RefuelAmountKg = nullableFloatPointer(refuelAmount) result.ChargeEnergyKWh = nullableFloatPointer(chargeEnergy) @@ -1267,7 +1268,7 @@ func (s *ProductionStore) MileageStatistics(ctx context.Context, query url.Value result := MileageStatistics{ DateFrom: query.Get("dateFrom"), DateTo: query.Get("dateTo"), Trend: []MileageTrendPoint{}, Ranking: []MileageVehicleRank{}, - Evidence: "vehicle_daily_mileage（按车辆和日期去重；百公里氢耗按匹配车辆日的纯氢里程计算）/ vehicle_open_daily_energy（质量通过的日用氢量）/ vehicle_realtime_location（最新里程表）", + Evidence: "vehicle_daily_mileage（按车辆和日期去重；百公里氢耗按匹配车辆日的纯氢里程计算）/ vehicle_open_daily_energy（质量通过的日修正用氢量）/ vehicle_realtime_location（最新里程表）", } summary := buildMileageStatisticsSummarySQL(query) if err := s.db.QueryRowContext(ctx, summary.Text, summary.Args...).Scan( diff --git a/vehicle-data-platform/apps/api/internal/platform/production_store_test.go b/vehicle-data-platform/apps/api/internal/platform/production_store_test.go index 94fdf3a6..1ba02bbf 100644 --- a/vehicle-data-platform/apps/api/internal/platform/production_store_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/production_store_test.go @@ -24,14 +24,14 @@ func TestProductionStoreDailyMileageReturnsSuspectHydrogenExportDetails(t *testi "hydrogen_consumption_kg", "hydrogen_consumption_kg_per_100km", "hydrogen_soc_balanced_kg", "hydrogen_soc_balanced_kg_per_100km", "pure_electric_mileage_km", "mixed_mileage_km", "battery_soc_delta_pct", "charge_count", "charge_energy_kwh", "refuel_count", "refuel_amount_kg", "hydrogen_evidence_available", "hydrogen_quality_status", "hydrogen_quality_reason", - "hydrogen_algorithm_version", "protocol", + "hydrogen_algorithm_version", "protocol", "province", "city", "region", "location_time", "location_status", } mock.ExpectQuery("SELECT m.vin"). WithArgs(20, 0). WillReturnRows(sqlmock.NewRows(columns).AddRow( "LB9A32A29R0LS1423", "粤AGR9816", "2026-08-26", 23119.8, 23333.1, 213.3, 213.3, 4.491, 2.106, 4.544, 2.130, 0.0, 213.3, -4.0, 1, 12.5, - 1, 8.976, 1, "SUSPECT", "疑似管路泄压", "PRESSURE_NIST_VALID_BOUNDARY_CHARGE_CYCLE_V3_5", "GB32960", + 1, 8.976, 1, "SUSPECT", "疑似管路泄压", "PRESSURE_NIST_VALID_BOUNDARY_CHARGE_CYCLE_V3_5", "GB32960", "广东省", "广州市", "华南", "2026-08-26T23:59:00+08:00", "已解析", )) page, err := (&ProductionStore{db: db}).DailyMileage(context.Background(), url.Values{"skipCount": {"1"}}) @@ -42,6 +42,9 @@ func TestProductionStoreDailyMileageReturnsSuspectHydrogenExportDetails(t *testi t.Fatalf("items=%#v", page.Items) } row := page.Items[0] + if row.Province != "广东省" || row.City != "广州市" || row.Region != "华南" || row.LocationStatus != "已解析" { + t.Fatalf("missing persisted geography: %+v", row) + } if row.HydrogenConsumptionKg == nil || *row.HydrogenConsumptionKg != 4.491 || row.HydrogenQualityStatus != "SUSPECT" || row.HydrogenQualityReason == "" || row.ChargeCount == nil || *row.ChargeCount != 1 || row.ChargeEnergyKWh == nil || *row.ChargeEnergyKWh != 12.5 || row.RefuelCount == nil || *row.RefuelCount != 1 || row.RefuelAmountKg == nil || *row.RefuelAmountKg != 8.976 { diff --git a/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go b/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go index 1ad34dae..08105b92 100644 --- a/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/query_builders_test.go @@ -693,7 +693,7 @@ func TestBuildMileageStatisticsSQLDeduplicatesVehicleDays(t *testing.T) { "GROUP BY m.vin, m.stat_date", "MAX(COALESCE(m.daily_mileage_km", "MAX(COALESCE(m.pure_hydrogen_mileage_km", - "MAX(h.consumption_kg)", + "MAX(h.soc_balanced_consumption_kg)", "COUNT(DISTINCT d.vin)", "SUM(d.daily_mileage_km)", "SUM(d.pure_hydrogen_mileage_km)", diff --git a/vehicle-data-platform/apps/api/internal/platform/service.go b/vehicle-data-platform/apps/api/internal/platform/service.go index b4860da9..04adb658 100644 --- a/vehicle-data-platform/apps/api/internal/platform/service.go +++ b/vehicle-data-platform/apps/api/internal/platform/service.go @@ -5591,10 +5591,18 @@ func (s *Service) DailyMileage(ctx context.Context, query url.Values) (Page[Dail if err != nil { return Page[DailyMileageRow]{}, err } + for index := range result.Items { + row := &result.Items[index] + row.HydrogenPhysicalConsumptionKg = row.HydrogenConsumptionKg + row.HydrogenConsumptionKg = row.HydrogenSOCBalancedKg + row.HydrogenConsumptionKgPer100Km = correctedDailyHydrogenRate(*row) + } + if !hydrogenConsumptionAllowed(ctx) { for index := range result.Items { result.Items[index].PureHydrogenMileageKm = 0 result.Items[index].HydrogenConsumptionKg = nil + result.Items[index].HydrogenPhysicalConsumptionKg = nil result.Items[index].HydrogenConsumptionKgPer100Km = nil result.Items[index].HydrogenSOCBalancedKg = nil result.Items[index].HydrogenSOCBalancedKgPer100Km = nil @@ -6702,3 +6710,16 @@ func boolToInt(value bool) int { } return 0 } + +// correctedDailyHydrogenRate keeps physical consumption available for audit, +// but never uses it as a fallback for the displayed consumption rate. +func correctedDailyHydrogenRate(row DailyMileageRow) *float64 { + if row.HydrogenSOCBalancedKg == nil { + return nil + } + mileage := row.PureHydrogenMileageKm + if row.MixedMileageKm != nil && *row.MixedMileageKm > 0 { + mileage = *row.MixedMileageKm + } + return hydrogenRatePer100Km(*row.HydrogenSOCBalancedKg, mileage, 1) +} diff --git a/vehicle-data-platform/apps/api/internal/platform/service_test.go b/vehicle-data-platform/apps/api/internal/platform/service_test.go index fa04c9e0..fd927212 100644 --- a/vehicle-data-platform/apps/api/internal/platform/service_test.go +++ b/vehicle-data-platform/apps/api/internal/platform/service_test.go @@ -629,7 +629,7 @@ func TestHydrogenConsumptionMetricsAreNotExposedToCustomerAccounts(t *testing.T) t.Fatal(err) } if len(daily.Items) != 1 || daily.Items[0].PureHydrogenMileageKm != 0 || daily.Items[0].HydrogenConsumptionKg != nil || daily.Items[0].HydrogenConsumptionKgPer100Km != nil || - daily.Items[0].ChargeEnergyKWh != nil || daily.Items[0].RefuelAmountKg != nil || daily.Items[0].HydrogenQualityStatus != "" || daily.Items[0].HydrogenEvidenceAvailable { + daily.Items[0].HydrogenPhysicalConsumptionKg != nil || daily.Items[0].ChargeEnergyKWh != nil || daily.Items[0].RefuelAmountKg != nil || daily.Items[0].HydrogenQualityStatus != "" || daily.Items[0].HydrogenEvidenceAvailable { t.Fatalf("customer daily mileage exposed hydrogen metrics: %+v", daily.Items) } summary, err := service.MileageStatistics(customer, query) diff --git a/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/main.go b/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/main.go new file mode 100644 index 00000000..a06b8d5b --- /dev/null +++ b/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/main.go @@ -0,0 +1,681 @@ +package main + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "lingniu/vehicle-data-platform/apps/api/internal/openplatform" +) + +const ( + baseURL = "http://115.29.187.205:20200" + hydrogenKWh = 16.0 +) + +type candidate struct { + VIN string `json:"vin"` + StatDate string `json:"stat_date"` + PlatformName string `json:"platform_name"` + DailyMileageKm float64 `json:"daily_mileage_km"` + RawTotal int `json:"raw_total"` + Plate string `json:"plate,omitempty"` + SampleNo int `json:"sample_no,omitempty"` + MileageBin int `json:"mileage_bin,omitempty"` + Model string `json:"model,omitempty"` + TankCapacityL float64 `json:"tank_capacity_l,omitempty"` + BatteryKWh float64 `json:"battery_capacity_kwh,omitempty"` +} + +type capacityRecord struct { + VIN, Plate, Model string + TankCapacityL float64 + Active int +} + +type rawFrame struct { + TS string `json:"ts"` + FrameID string `json:"frame_id"` + EventID string `json:"event_id"` + MessageID int `json:"message_id"` + MessageIDHex string `json:"message_id_hex"` + EventTime string `json:"event_time"` + ReceivedAt string `json:"received_at"` + RawSizeBytes int `json:"raw_size_bytes"` + RawHex string `json:"raw_hex"` + ParsedFields map[string]any `json:"parsed_fields"` + ParseStatus string `json:"parse_status"` + SourceEndpoint string `json:"source_endpoint"` + Protocol string `json:"protocol"` + VIN string `json:"vin"` +} + +type rawResponse struct { + Items []rawFrame `json:"items"` + Total int `json:"total"` +} + +type dayOutput struct { + Candidate candidate `json:"candidate"` + APIRawFrameCount int `json:"apiRawFrameCount"` + UniqueFrameCount int `json:"uniqueFrameCount"` + DuplicateFrameCount int `json:"duplicateFrameCount"` + AlgorithmSamples int `json:"algorithmSamples"` + CriticalFieldRows int `json:"criticalFieldRows"` + EarliestEventTime string `json:"earliestEventTime"` + LatestEventTime string `json:"latestEventTime"` + RawArchiveFile string `json:"rawArchiveFile"` + RawArchiveSHA256 string `json:"rawArchiveSha256"` + RawArchiveBytes int64 `json:"rawArchiveBytes"` + Stat openplatform.HydrogenDailyStat `json:"stat"` +} + +func main() { + if len(os.Args) != 5 && len(os.Args) != 6 { + panic("usage: audit [source-archive-dir]") + } + metadataPath, startDate, endDate, outputDir := os.Args[1], os.Args[2], os.Args[3], os.Args[4] + sourceArchiveDir := "" + if len(os.Args) == 6 { + sourceArchiveDir = os.Args[5] + } + start, err := time.Parse("2006-01-02", startDate) + if err != nil { + panic(err) + } + end, err := time.Parse("2006-01-02", endDate) + if err != nil || end.Before(start) { + panic("invalid date range") + } + var metadata []struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + Model string `json:"model"` + TankCapacityL float64 `json:"tank_capacity_l"` + BatteryCapacityKWh float64 `json:"battery_capacity_kwh"` + } + mustReadJSON(metadataPath, &metadata) + if len(metadata) == 0 { + panic("vehicle metadata is empty") + } + selected := make([]candidate, 0, len(metadata)*int(end.Sub(start).Hours()/24+1)) + for day := start; !day.After(end); day = day.AddDate(0, 0, 1) { + statDate := day.Format("2006-01-02") + for _, item := range metadata { + if len(strings.TrimSpace(item.VIN)) != 17 || item.TankCapacityL <= 0 || item.BatteryCapacityKWh <= 0 { + continue + } + selected = append(selected, candidate{ + VIN: strings.ToUpper(strings.TrimSpace(item.VIN)), StatDate: statDate, + Plate: item.Plate, Model: item.Model, TankCapacityL: item.TankCapacityL, + BatteryKWh: item.BatteryCapacityKWh, + }) + } + } + sort.Slice(selected, func(i, j int) bool { + if selected[i].StatDate != selected[j].StatDate { + return selected[i].StatDate < selected[j].StatDate + } + if selected[i].Plate != selected[j].Plate { + return selected[i].Plate < selected[j].Plate + } + return selected[i].VIN < selected[j].VIN + }) + for index := range selected { + selected[index].SampleNo = index + 1 + } + if sourceArchiveDir == "" { + for day := start; !day.After(end); day = day.AddDate(0, 0, 1) { + if err := os.MkdirAll(filepath.Join(outputDir, "raw", day.Format("2006-01-02")), 0o755); err != nil { + panic(err) + } + } + } + writeJSON(filepath.Join(outputDir, "selected_vehicles.json"), selected) + + jobs := make(chan candidate) + results := make(chan dayOutput) + errs := make(chan error, len(selected)) + var wg sync.WaitGroup + for i := 0; i < 12; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for c := range jobs { + out, err := processDay(c, outputDir, sourceArchiveDir) + if err != nil { + errs <- err + continue + } + results <- out + } + }() + } + go func() { + for _, c := range selected { + jobs <- c + } + close(jobs) + wg.Wait() + close(results) + close(errs) + }() + outputs := make([]dayOutput, 0, len(selected)) + for out := range results { + if out.APIRawFrameCount == 0 { + continue + } + outputs = append(outputs, out) + fmt.Fprintf(os.Stderr, "completed %03d %s %s raw=%d samples=%d quality=%s\n", out.Candidate.SampleNo, out.Candidate.Plate, out.Candidate.VIN, out.APIRawFrameCount, out.AlgorithmSamples, out.Stat.QualityStatus) + } + var allErrs []string + for err := range errs { + allErrs = append(allErrs, err.Error()) + } + if len(allErrs) > 0 { + panic(strings.Join(allErrs, "\n")) + } + sort.Slice(outputs, func(i, j int) bool { return outputs[i].Candidate.SampleNo < outputs[j].Candidate.SampleNo }) + writeJSON(filepath.Join(outputDir, "daily_results.json"), outputs) + writeManifest(filepath.Join(outputDir, "raw_manifest.csv"), outputs) + fmt.Printf("processed=%d raw_frames=%d algorithm_samples=%d intervals=%d\n", len(outputs), sumRaw(outputs), sumSamples(outputs), sumIntervals(outputs)) +} + +func stratifiedSelect(all []candidate, plateByVIN map[string]string, capacityByVIN map[string]capacityRecord) []candidate { + general, cold := make([]candidate, 0), make([]candidate, 0) + for _, c := range all { + cap, ok := capacityByVIN[strings.ToUpper(c.VIN)] + if !ok || cap.Active != 1 || c.DailyMileageKm < 10 || c.DailyMileageKm > 600 || c.RawTotal < 300 { + continue + } + if cap.Model != "4.5吨货车" && cap.Model != "帕力安牌4.5吨冷链车" { + continue + } + c.Plate = cap.Plate + if c.Plate == "" { + c.Plate = plateByVIN[strings.ToUpper(c.VIN)] + } + c.Model = cap.Model + c.TankCapacityL = cap.TankCapacityL + if cap.Model == "4.5吨货车" { + general = append(general, c) + } else { + cold = append(cold, c) + } + } + if len(general) < 41 || len(cold) < 59 { + panic(fmt.Sprintf("insufficient pools general=%d cold=%d", len(general), len(cold))) + } + selected := append(selectEvenly(general, 41), selectEvenly(cold, 59)...) + sort.Slice(selected, func(i, j int) bool { + if selected[i].DailyMileageKm != selected[j].DailyMileageKm { + return selected[i].DailyMileageKm < selected[j].DailyMileageKm + } + return selected[i].VIN < selected[j].VIN + }) + for i := range selected { + selected[i].SampleNo = i + 1 + selected[i].MileageBin = i/10 + 1 + } + return selected +} + +func selectEvenly(values []candidate, count int) []candidate { + sort.Slice(values, func(i, j int) bool { + if values[i].DailyMileageKm != values[j].DailyMileageKm { + return values[i].DailyMileageKm < values[j].DailyMileageKm + } + return values[i].VIN < values[j].VIN + }) + out := make([]candidate, 0, count) + for i := 0; i < count; i++ { + idx := 0 + if count > 1 { + idx = int(math.Round(float64(i) * float64(len(values)-1) / float64(count-1))) + } + out = append(out, values[idx]) + } + return out +} + +func processDay(c candidate, outputDir, sourceArchiveDir string) (dayOutput, error) { + archiveName := rawArchiveName(c) + archivePath := filepath.Join(outputDir, "raw", c.StatDate, archiveName) + readArchivePath := archivePath + if sourceArchiveDir != "" { + readArchivePath = filepath.Join(sourceArchiveDir, "raw", c.StatDate, archiveName) + if _, statErr := os.Stat(readArchivePath); os.IsNotExist(statErr) { + // Sample numbers depend on the selected vehicle/date set. Reuse an + // existing archive by its stable plate/VIN/date suffix when rerunning + // only a small subset for validation. + pattern := filepath.Join(sourceArchiveDir, "raw", c.StatDate, "*_"+c.Plate+"_"+c.VIN+"_"+c.StatDate+".csv.gz") + if matches, _ := filepath.Glob(pattern); len(matches) == 1 { + readArchivePath = matches[0] + } + } + } + frames := []rawFrame(nil) + total := 0 + var err error + if _, statErr := os.Stat(readArchivePath); statErr == nil { + frames, err = readRawArchive(readArchivePath) + total = len(frames) + } else if sourceArchiveDir != "" && os.IsNotExist(statErr) { + return dayOutput{Candidate: c}, nil + } else { + for attempt := 1; attempt <= 3; attempt++ { + frames, total, err = fetchFrames(c.VIN, c.StatDate) + if err == nil { + break + } + if attempt < 3 { + time.Sleep(time.Duration(attempt) * time.Second) + } + } + } + if err != nil { + return dayOutput{}, fmt.Errorf("%s %s: %w", c.StatDate, c.VIN, err) + } + if total == 0 { + return dayOutput{Candidate: c}, nil + } + sort.SliceStable(frames, func(i, j int) bool { + if frames[i].EventTime != frames[j].EventTime { + return frames[i].EventTime < frames[j].EventTime + } + if frames[i].SourceEndpoint != frames[j].SourceEndpoint { + return frames[i].SourceEndpoint < frames[j].SourceEndpoint + } + return frames[i].TS < frames[j].TS + }) + seen := map[string]bool{} + duplicateFrames := 0 + observations := make([]openplatform.HydrogenObservation, 0, len(frames)) + criticalRows := 0 + for _, f := range frames { + if f.FrameID != "" { + if seen[f.FrameID] { + duplicateFrames++ + } + seen[f.FrameID] = true + } + obs, ok, critical := observationFromFrame(f, c.TankCapacityL, c.StatDate) + if critical { + criticalRows++ + } + if ok { + observations = append(observations, obs) + } + } + params := map[string]openplatform.HydrogenCalculationParameters{c.VIN: {BatteryCapacityKWh: c.BatteryKWh, HydrogenEnergyKWhKg: hydrogenKWh}} + stats := openplatform.BuildHydrogenDailyStatsOrderedWithParameters(observations, c.StatDate, 0.05, 20, params) + var stat openplatform.HydrogenDailyStat + if len(stats) == 1 { + stat = stats[0] + } else { + stat = openplatform.HydrogenDailyStat{VIN: c.VIN, Date: c.StatDate, SampleCount: len(observations), QualityStatus: "NO_DATA", QualityReason: "无有效压力温度样本"} + } + roles := map[string]string{} + for _, interval := range stat.Intervals { + appendRole(roles, interval.StartEventID, fmt.Sprintf("运行区间%d(%s)起点", interval.Index, interval.Type)) + appendRole(roles, interval.EndEventID, fmt.Sprintf("运行区间%d(%s)终点", interval.Index, interval.Type)) + } + for _, interval := range stat.HydrogenIntervals { + appendRole(roles, interval.StartEventID, fmt.Sprintf("氢量分段%d起点", interval.Index)) + appendRole(roles, interval.EndEventID, fmt.Sprintf("氢量分段%d终点", interval.Index)) + } + // Even when the original frames are reused, rewrite the audit CSV so its + // “计算角色/排除原因” column matches the current algorithm version and + // interval boundaries. The original HEX and parsed fields remain unchanged. + if sourceArchiveDir == "" { + if err := writeRawCSV(archivePath, c, frames, roles); err != nil { + return dayOutput{}, err + } + } + checksum, size, err := fileSHA256(readArchivePath) + if err != nil { + return dayOutput{}, err + } + out := dayOutput{Candidate: c, APIRawFrameCount: total, UniqueFrameCount: len(seen), DuplicateFrameCount: duplicateFrames, AlgorithmSamples: len(observations), CriticalFieldRows: criticalRows, RawArchiveFile: filepath.ToSlash(filepath.Join("raw", c.StatDate, archiveName)), RawArchiveSHA256: checksum, RawArchiveBytes: size, Stat: stat} + if len(frames) > 0 { + out.EarliestEventTime = frames[0].EventTime + out.LatestEventTime = frames[len(frames)-1].EventTime + } + return out, nil +} + +func rawArchiveName(c candidate) string { + return fmt.Sprintf("%04d_%s_%s_%s.csv.gz", c.SampleNo, safeName(c.Plate), c.VIN, c.StatDate) +} + +func appendRole(roles map[string]string, eventID, role string) { + if eventID == "" { + return + } + if roles[eventID] == "" { + roles[eventID] = role + return + } + roles[eventID] += "；" + role +} + +func fileSHA256(path string) (string, int64, error) { + file, err := os.Open(path) + if err != nil { + return "", 0, err + } + defer file.Close() + hash := sha256.New() + size, err := io.Copy(hash, file) + if err != nil { + return "", 0, err + } + return fmt.Sprintf("%x", hash.Sum(nil)), size, nil +} + +func readRawArchive(path string) ([]rawFrame, error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + defer file.Close() + gz, err := gzip.NewReader(file) + if err != nil { + return nil, err + } + defer gz.Close() + reader := csv.NewReader(gz) + rows, err := reader.ReadAll() + if err != nil { + return nil, err + } + if len(rows) == 0 { + return nil, fmt.Errorf("empty raw archive %s", path) + } + columns := map[string]int{} + for index, name := range rows[0] { + columns[name] = index + } + cell := func(row []string, name string) string { + index, ok := columns[name] + if !ok || index >= len(row) { + return "" + } + return row[index] + } + frames := make([]rawFrame, 0, len(rows)-1) + for _, row := range rows[1:] { + fields := map[string]any{} + if rawFields := cell(row, "完整解析字段JSON"); rawFields != "" { + _ = json.Unmarshal([]byte(rawFields), &fields) + } + messageID, _ := strconv.Atoi(cell(row, "消息ID")) + rawSize, _ := strconv.Atoi(cell(row, "原始字节数")) + frames = append(frames, rawFrame{ + FrameID: cell(row, "帧ID"), EventID: cell(row, "事件ID"), MessageID: messageID, + MessageIDHex: cell(row, "消息ID_HEX"), EventTime: cell(row, "事件时间"), ReceivedAt: cell(row, "接收时间"), + RawSizeBytes: rawSize, RawHex: cell(row, "原始报文HEX"), ParsedFields: fields, + ParseStatus: cell(row, "解析状态"), SourceEndpoint: cell(row, "源端点"), Protocol: "GB32960", VIN: cell(row, "VIN"), + }) + } + return frames, nil +} + +func fetchFrames(vin, statDate string) ([]rawFrame, int, error) { + const limit = 500 + first, err := fetchPage(vin, statDate, 0, limit, true) + if err != nil { + return nil, 0, err + } + frames := append([]rawFrame(nil), first.Items...) + for offset := limit; offset < first.Total; offset += limit { + page, err := fetchPage(vin, statDate, offset, limit, false) + if err != nil { + return nil, 0, err + } + frames = append(frames, page.Items...) + } + if len(frames) != first.Total { + return nil, first.Total, fmt.Errorf("API total=%d fetched=%d", first.Total, len(frames)) + } + return frames, first.Total, nil +} + +func fetchPage(vin, statDate string, offset, limit int, includeTotal bool) (rawResponse, error) { + q := url.Values{"protocol": {"GB32960"}, "vin": {vin}, "dateFrom": {statDate + " 00:00:00"}, "dateTo": {statDate + " 23:59:59"}, "orderBy": {"eventTime"}, "limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(offset)}, "includeFields": {"true"}, "includePayload": {"true"}, "includeTotal": {strconv.FormatBool(includeTotal)}} + req, _ := http.NewRequest(http.MethodGet, baseURL+"/api/history/raw-frames?"+q.Encode(), nil) + req.Header.Set("Accept-Encoding", "gzip") + client := &http.Client{Timeout: 90 * time.Second} + res, err := client.Do(req) + if err != nil { + return rawResponse{}, err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + if res.StatusCode == http.StatusInternalServerError && bytes.Contains(body, []byte("Table does not exist")) { + return rawResponse{}, nil + } + return rawResponse{}, fmt.Errorf("HTTP %d: %s", res.StatusCode, body) + } + var reader io.Reader = res.Body + if res.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(res.Body) + if err != nil { + return rawResponse{}, err + } + defer gz.Close() + reader = gz + } + var out rawResponse + if err := json.NewDecoder(reader).Decode(&out); err != nil { + return rawResponse{}, err + } + return out, nil +} + +func observationFromFrame(f rawFrame, tankCapacity float64, statDate string) (openplatform.HydrogenObservation, bool, bool) { + if f.MessageID != 2 || f.ParseStatus != "OK" { + return openplatform.HydrogenObservation{}, false, false + } + pressure, pok := num(f.ParsedFields["gb32960.fuel_cell.max_hydrogen_pressure_mpa"]) + temp, tok := num(f.ParsedFields["gb32960.fuel_cell.max_hydrogen_temperature_c"]) + critical := pok && tok + if !pok || !tok || pressure <= 0 || pressure > 70 || temp <= -40 || temp > 726.85 { + return openplatform.HydrogenObservation{}, false, critical + } + mass, ok := openplatform.PressureHydrogenMassKg(pressure, temp, tankCapacity) + if !ok { + return openplatform.HydrogenObservation{}, false, critical + } + step, _ := openplatform.PressureHydrogenMassKg(math.Max(0, pressure-0.2), temp, tankCapacity) + parsed, _ := json.Marshal(f.ParsedFields) + _, _, active, known, _ := openplatform.ExtractHydrogenTelemetry(string(parsed)) + t, err := parseEventTime(f.EventTime) + if err != nil || t.Format("2006-01-02") != statDate { + return openplatform.HydrogenObservation{}, false, critical + } + o := openplatform.HydrogenObservation{VIN: f.VIN, Source: f.SourceEndpoint, EventID: f.EventID, ObservedAt: t, MassKg: mass, TankCapacityLiter: tankCapacity, PressureMPa: pressure, TemperatureC: temp, NoiseKg: math.Min(1, math.Max(0.05, mass-step)), RefuelThresholdKg: math.Max(1, mass*0.05), FuelCellActive: active, FuelCellStateKnown: known} + if voltage, vok := num(f.ParsedFields["gb32960.fuel_cell.fuel_cell_voltage_v"]); vok && voltage > 0 && voltage <= 1000 { + if current, cok := num(f.ParsedFields["gb32960.fuel_cell.fuel_cell_current_a"]); cok && current >= 0 && current <= 2000 { + o.FuelCellVoltageV = voltage + o.FuelCellCurrentA = current + o.FuelCellPowerKnown = true + } + } + if v, ok := num(f.ParsedFields["gb32960.vehicle.soc_percent"]); ok && v >= 0 && v <= 100 { + o.SOCPercent = v + o.SOCKnown = true + } + if v, ok := num(f.ParsedFields["gb32960.vehicle.total_mileage_km"]); ok && v >= 0 { + o.MileageKm = v + o.MileageKnown = true + } + if v, ok := num(f.ParsedFields["gb32960.vehicle.vehicle_status"]); ok { + o.VehicleState = int(v) + o.VehicleStateKnown = v >= 0 && v <= 255 + } + if v, ok := num(f.ParsedFields["gb32960.vehicle.charge_status"]); ok { + o.ChargeState = int(v) + o.ChargeStateKnown = v >= 0 && v <= 255 + } + if v, ok := num(f.ParsedFields["gb32960.vehicle.running_mode"]); ok { + o.RunningMode = int(v) + o.RunningModeKnown = v >= 0 && v <= 255 + } + return o, true, critical +} + +var rawHeaders = []string{"序号", "车牌", "VIN", "统计日期", "车型", "储氢总容积(L)", "额定电量(kWh)", "事件时间", "接收时间", "帧ID", "事件ID", "消息ID", "消息ID_HEX", "源端点", "解析状态", "原始字节数", "原始报文HEX", "完整解析字段JSON", "最高氢压(MPa)", "最高氢温(℃)", "电池SOC(%)", "仪表总里程(km)", "车辆状态", "充电状态", "运行模式", "燃料电池工作状态", "燃料电池电流(A)", "车端氢气质量(kg)", "系统压力换算剩余氢量(kg)", "噪声阈值(kg)", "是否算法有效样本", "计算角色/排除原因"} + +func writeRawCSV(path string, c candidate, frames []rawFrame, roles map[string]string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + gz := gzip.NewWriter(f) + defer gz.Close() + w := csv.NewWriter(gz) + defer w.Flush() + if err := w.Write(rawHeaders); err != nil { + return err + } + for _, rf := range frames { + o, valid, _ := observationFromFrame(rf, c.TankCapacityL, c.StatDate) + role := roles[rf.EventID] + if !valid { + if t, err := parseEventTime(rf.EventTime); err == nil && t.Format("2006-01-02") != c.StatDate { + role = "事件时间不在统计日，不进入计算" + } else if rf.MessageID != 2 { + role = "非实时信息上报帧，不进入计算" + } else if rf.ParseStatus != "OK" { + role = "解析状态非OK，不进入计算" + } else { + role = "压力/温度无效或缺失，不进入计算" + } + } else if role == "" { + role = "有效候选帧（由分段规则判定）" + } + parsedJSON, _ := json.Marshal(rf.ParsedFields) + row := []string{fmt.Sprintf("%04d", c.SampleNo), c.Plate, c.VIN, c.StatDate, c.Model, strconv.FormatFloat(c.TankCapacityL, 'f', 2, 64), strconv.FormatFloat(c.BatteryKWh, 'f', 2, 64), rf.EventTime, rf.ReceivedAt, rf.FrameID, rf.EventID, strconv.Itoa(rf.MessageID), rf.MessageIDHex, rf.SourceEndpoint, rf.ParseStatus, strconv.Itoa(rf.RawSizeBytes), rf.RawHex, string(parsedJSON), val(rf.ParsedFields, "gb32960.fuel_cell.max_hydrogen_pressure_mpa"), val(rf.ParsedFields, "gb32960.fuel_cell.max_hydrogen_temperature_c"), val(rf.ParsedFields, "gb32960.vehicle.soc_percent"), val(rf.ParsedFields, "gb32960.vehicle.total_mileage_km"), val(rf.ParsedFields, "gb32960.vehicle.vehicle_status"), val(rf.ParsedFields, "gb32960.vehicle.charge_status"), val(rf.ParsedFields, "gb32960.vehicle.running_mode"), val(rf.ParsedFields, "gb32960.gd_fc_stack.engine_work_state"), val(rf.ParsedFields, "gb32960.fuel_cell.fuel_cell_current_a"), val(rf.ParsedFields, "gb32960.gd_fc_vehicle_info.hydrogen_mass_kg"), blankFloat(valid, o.MassKg), blankFloat(valid, o.NoiseKg), yesNo(valid), role} + if err := w.Write(row); err != nil { + return err + } + } + return w.Error() +} + +func writeManifest(path string, outputs []dayOutput) { + f, err := os.Create(path) + if err != nil { + panic(err) + } + defer f.Close() + w := csv.NewWriter(f) + defer w.Flush() + _ = w.Write([]string{"序号", "车牌", "VIN", "日期", "车型", "储氢总容积(L)", "额定电量(kWh)", "API原始帧数", "唯一帧数", "重复帧数", "算法样本数", "关键字段帧数", "最早事件时间", "最晚事件时间", "原始文件", "压缩包SHA256", "压缩字节数", "质量状态", "质量原因"}) + for _, o := range outputs { + _ = w.Write([]string{fmt.Sprintf("%04d", o.Candidate.SampleNo), o.Candidate.Plate, o.Candidate.VIN, o.Candidate.StatDate, o.Candidate.Model, strconv.FormatFloat(o.Candidate.TankCapacityL, 'f', 2, 64), strconv.FormatFloat(o.Candidate.BatteryKWh, 'f', 2, 64), strconv.Itoa(o.APIRawFrameCount), strconv.Itoa(o.UniqueFrameCount), strconv.Itoa(o.DuplicateFrameCount), strconv.Itoa(o.AlgorithmSamples), strconv.Itoa(o.CriticalFieldRows), o.EarliestEventTime, o.LatestEventTime, o.RawArchiveFile, o.RawArchiveSHA256, strconv.FormatInt(o.RawArchiveBytes, 10), o.Stat.QualityStatus, o.Stat.QualityReason}) + } +} + +func num(v any) (float64, bool) { + switch x := v.(type) { + case float64: + return x, true + case string: + n, e := strconv.ParseFloat(strings.TrimSpace(x), 64) + return n, e == nil + case json.Number: + n, e := x.Float64() + return n, e == nil + default: + return 0, false + } +} +func val(m map[string]any, k string) string { + if v, ok := m[k]; ok { + return fmt.Sprint(v) + } + return "" +} +func blankFloat(ok bool, v float64) string { + if !ok { + return "" + } + return strconv.FormatFloat(v, 'f', 6, 64) +} +func yesNo(v bool) string { + if v { + return "是" + } + return "否" +} +func safeName(v string) string { + v = strings.TrimSpace(v) + if v == "" { + return "无车牌" + } + return strings.NewReplacer("/", "_", "\\", "_", " ", "_").Replace(v) +} +func parseEventTime(value string) (time.Time, error) { + loc := time.FixedZone("CST", 8*3600) + if t, e := time.ParseInLocation("2006-01-02 15:04:05.000", value, loc); e == nil { + return t, nil + } + return time.ParseInLocation("2006-01-02 15:04:05", value, loc) +} +func mustReadJSON(path string, v any) { + b, e := os.ReadFile(path) + if e != nil { + panic(e) + } + if e = json.Unmarshal(b, v); e != nil { + panic(e) + } +} +func writeJSON(path string, v any) { + b, e := json.MarshalIndent(v, "", " ") + if e != nil { + panic(e) + } + if e = os.WriteFile(path, b, 0o644); e != nil { + panic(e) + } +} +func sumRaw(v []dayOutput) int { + n := 0 + for _, x := range v { + n += x.APIRawFrameCount + } + return n +} +func sumSamples(v []dayOutput) int { + n := 0 + for _, x := range v { + n += x.AlgorithmSamples + } + return n +} +func sumIntervals(v []dayOutput) int { + n := 0 + for _, x := range v { + n += len(x.Stat.Intervals) + } + return n +} diff --git a/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/query_capacity.go b/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/query_capacity.go new file mode 100644 index 00000000..fb5e3d6e --- /dev/null +++ b/vehicle-data-platform/apps/api/tmp/hydrogen-audit-100-real/query_capacity.go @@ -0,0 +1,38 @@ +//go:build ignore + +package main + +import ( + "database/sql" + "encoding/json" + "fmt" + "net" + "net/url" + "os" + "sort" + "strings" + "time" + + _ "github.com/go-sql-driver/mysql" +) + +func main() { + if len(os.Args)!=3 { panic("usage: query_capacity ") } + var selected []struct{ VIN string `json:"vin"` } + b,err:=os.ReadFile(os.Args[1]);if err!=nil{panic(err)};if err=json.Unmarshal(b,&selected);err!=nil{panic(err)} + prefix := "HYDROGEN_DB_" + host := strings.TrimSpace(os.Getenv(prefix + "HOST")) + port := strings.TrimSpace(os.Getenv(prefix + "PORT")); if port == "" { port = "3306" } + name := strings.TrimSpace(os.Getenv(prefix + "NAME")) + user := os.Getenv(prefix + "USER"); pass := os.Getenv(prefix + "PASSWORD") + dsn := user+":"+pass+"@tcp("+net.JoinHostPort(host,port)+")/"+name+"?parseTime=true&loc="+url.QueryEscape("Asia/Shanghai")+"&timeout=5s&readTimeout=15s" + db,err:=sql.Open("mysql",dsn); if err!=nil{panic(err)}; defer db.Close(); db.SetConnMaxLifetime(time.Minute); if err=db.Ping();err!=nil{panic(err)} + vins:=make([]string,0,len(selected));args:=make([]any,0,len(selected));for _,s:=range selected{v:=strings.ToUpper(strings.TrimSpace(s.VIN));vins=append(vins,v);args=append(args,v)} + placeholders:=strings.TrimRight(strings.Repeat("?,",len(vins)),",") + rows,err:=db.Query(`SELECT UPPER(TRIM(vin)),plate_number,brand_name,model_name,tank_capacity_l,active,source_model_id,source_updated_at,synced_at FROM lingniu_vehicle_data.vehicle_hydrogen_tank_capacity WHERE UPPER(TRIM(vin)) IN (`+placeholders+`) ORDER BY vin`,args...);if err!=nil{panic(err)};defer rows.Close() + type row struct{VIN,Plate,Brand,Model string;TankCapacityL float64;Active int;SourceModelID *int64;SourceUpdatedAt, SyncedAt *time.Time} + out:=make([]row,0,len(vins));for rows.Next(){var r row;if err=rows.Scan(&r.VIN,&r.Plate,&r.Brand,&r.Model,&r.TankCapacityL,&r.Active,&r.SourceModelID,&r.SourceUpdatedAt,&r.SyncedAt);err!=nil{panic(err)};out=append(out,r)};if err=rows.Err();err!=nil{panic(err)} + sort.Slice(out,func(i,j int)bool{return out[i].VIN") + } + dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN")) + if dsn == "" { + panic("MYSQL_DSN is required") + } + db, err := sql.Open("mysql", dsn) + if err != nil { + panic(err) + } + defer db.Close() + rows, err := db.Query(`SELECT UPPER(TRIM(vi.vin)),COALESCE(vi.plate_number,''), +COALESCE(vm.brand,''),COALESCE(vm.model,''),vm.id,vm.battery_capacity,vm.tank_capacity, +CASE + WHEN vi.update_time IS NULL THEN vm.update_time + WHEN vm.update_time IS NULL THEN vi.update_time + ELSE GREATEST(vi.update_time,vm.update_time) +END +FROM ln_asset_management.vehicle_info vi +JOIN ln_asset_management.vehicle_model vm ON vm.id=vi.vehicle_model_id +WHERE COALESCE(vi.del_flag,'0')='0' AND COALESCE(vm.del_flag,'0')='0' + AND LOWER(TRIM(vm.brand))='hyundai' + AND LENGTH(TRIM(vi.vin))=17 + AND vm.tank_capacity>0 + AND vm.battery_capacity>0 +ORDER BY vi.vin,vi.update_time DESC,vi.id DESC`) + if err != nil { + panic(err) + } + defer rows.Close() + byVIN := map[string]metadataRecord{} + for rows.Next() { + var row metadataRecord + var updated sql.NullTime + if err := rows.Scan(&row.VIN, &row.Plate, &row.Brand, &row.Model, &row.VehicleModelID, &row.BatteryCapacityKWh, &row.TankCapacityL, &updated); err != nil { + panic(err) + } + if _, exists := byVIN[row.VIN]; exists { + continue + } + if updated.Valid { + value := updated.Time + row.SourceUpdatedAt = &value + } + byVIN[row.VIN] = row + } + if err := rows.Err(); err != nil { + panic(err) + } + vins := make([]string, 0, len(byVIN)) + for vin := range byVIN { + vins = append(vins, vin) + } + sort.Strings(vins) + out := make([]metadataRecord, 0, len(vins)) + for _, vin := range vins { + out = append(out, byVIN[vin]) + } + output, err := os.Create(os.Args[1]) + if err != nil { + panic(err) + } + defer output.Close() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(out); err != nil { + panic(err) + } +} diff --git a/vehicle-data-platform/apps/api/tmp/standard-model-trial/main.go b/vehicle-data-platform/apps/api/tmp/standard-model-trial/main.go new file mode 100644 index 00000000..deb54e84 --- /dev/null +++ b/vehicle-data-platform/apps/api/tmp/standard-model-trial/main.go @@ -0,0 +1,660 @@ +package main + +import ( + "bytes" + "compress/gzip" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" + + "lingniu/vehicle-data-platform/apps/api/internal/openplatform" +) + +const ( + baseURL = "http://115.29.187.205:20200" + hydrogenEnergyKWh = 16.0 + maxIntegrationGap = 30 * time.Second +) + +type vehicleMetadata struct { + VIN string `json:"vin"` + Plate string `json:"plate"` + Model string `json:"model"` + BatteryCapacityKWh float64 `json:"battery_capacity_kwh"` + TankCapacityL float64 `json:"tank_capacity_l"` +} + +type rawFrame struct { + TS string `json:"ts"` + FrameID string `json:"frame_id"` + EventID string `json:"event_id"` + MessageID int `json:"message_id"` + EventTime string `json:"event_time"` + ParseStatus string `json:"parse_status"` + SourceEndpoint string `json:"source_endpoint"` + VIN string `json:"vin"` + ParsedFields map[string]any `json:"parsed_fields"` +} + +type rawResponse struct { + Items []rawFrame `json:"items"` + Total int `json:"total"` +} + +type energySample struct { + At time.Time + VehicleStatus int + ChargeStatus int + SOC float64 + Mileage float64 + BatteryVoltageV float64 + BatteryCurrentA float64 + FuelCellVoltageV float64 + FuelCellCurrentA float64 + HasSOC bool + HasMileage bool + HasBatteryPower bool + HasFuelCellPower bool + HasVehicleStatus bool + HasChargeStatus bool +} + +type energyResult struct { + FirstTime string `json:"firstTime"` + LastTime string `json:"lastTime"` + StartSOC float64 `json:"startSoc"` + EndSOC float64 `json:"endSoc"` + SOCChangePctEndMinusStart float64 `json:"socChangePctEndMinusStart"` + StoredEnergyChangeKWhSOC float64 `json:"storedEnergyChangeKWhSoc"` + BatteryNetOutputKWhSOC float64 `json:"batteryNetOutputKWhSoc"` + BatteryNetOutputKWhIntegrated float64 `json:"batteryNetOutputKWhIntegrated"` + FuelCellOutputKWhIntegrated float64 `json:"fuelCellOutputKWhIntegrated"` + BatteryCoverageSeconds float64 `json:"batteryCoverageSeconds"` + FuelCellCoverageSeconds float64 `json:"fuelCellCoverageSeconds"` + OperatingSpanSeconds float64 `json:"operatingSpanSeconds"` + BatteryCoverageRatio float64 `json:"batteryCoverageRatio"` + FuelCellCoverageRatio float64 `json:"fuelCellCoverageRatio"` + ExternalChargeFrameCount int `json:"externalChargeFrameCount"` + EnergySampleCount int `json:"energySampleCount"` +} + +type row struct { + Date string `json:"date"` + Plate string `json:"plate"` + VIN string `json:"vin"` + Model string `json:"model"` + RawFrameCount int `json:"rawFrameCount"` + CurrentQuality string `json:"currentQuality"` + CurrentReason string `json:"currentReason"` + RefuelCount int `json:"refuelCount"` + ChargeCount int `json:"chargeCount"` + TotalMileageKm float64 `json:"totalMileageKm"` + PureElectricMileageKm float64 `json:"pureElectricMileageKm"` + CurrentMixedMileageKm float64 `json:"currentMixedMileageKm"` + PhysicalHydrogenKg float64 `json:"physicalHydrogenKg"` + CurrentSOCBalancedKg float64 `json:"currentSocBalancedKg"` + CurrentRatePerMixedKm float64 `json:"currentRatePerMixedKm"` + PhysicalRatePerTotalKm float64 `json:"physicalRatePerTotalKm"` + FullDayStartSOC float64 `json:"fullDayStartSoc"` + FullDayEndSOC float64 `json:"fullDayEndSoc"` + FullDaySOCChangePct float64 `json:"fullDaySocChangePct"` + StoredEnergyChangeKWhSOC float64 `json:"storedEnergyChangeKWhSoc"` + BatteryEquivalentKgFixed16 float64 `json:"batteryEquivalentKgFixed16"` + StandardLikeBalancedKg float64 `json:"standardLikeBalancedKg"` + StandardLikeRatePerTotalKm float64 `json:"standardLikeRatePerTotalKm"` + StandardLikeApplicable bool `json:"standardLikeApplicable"` + StandardLikeReason string `json:"standardLikeReason"` + BatteryNetOutputKWhIntegrated float64 `json:"batteryNetOutputKWhIntegrated"` + FuelCellOutputKWhIntegrated float64 `json:"fuelCellOutputKWhIntegrated"` + BatteryEnergyShare float64 `json:"batteryEnergyShare"` + FuelCellEnergyShare float64 `json:"fuelCellEnergyShare"` + BatteryContributionKm float64 `json:"batteryContributionKm"` + FuelCellContributionKm float64 `json:"fuelCellContributionKm"` + HydrogenRatePerFCContributionKm float64 `json:"hydrogenRatePerFcContributionKm"` + BatteryCoverageRatio float64 `json:"batteryCoverageRatio"` + FuelCellCoverageRatio float64 `json:"fuelCellCoverageRatio"` + ExternalChargeFrameCount int `json:"externalChargeFrameCount"` + PowerIntegrationUsable bool `json:"powerIntegrationUsable"` +} + +type job struct { + Vehicle vehicleMetadata + Date string +} + +func main() { + if len(os.Args) != 5 { + panic("usage: standard-model-trial ") + } + metadataPath, startDate, endDate, outputDir := os.Args[1], os.Args[2], os.Args[3], os.Args[4] + start := mustDate(startDate) + end := mustDate(endDate) + if end.Before(start) { + panic("end date is before start date") + } + var vehicles []vehicleMetadata + mustReadJSON(metadataPath, &vehicles) + if len(vehicles) == 0 { + panic("metadata is empty") + } + if err := os.MkdirAll(outputDir, 0o755); err != nil { + panic(err) + } + + jobs := make(chan job) + results := make(chan row) + errs := make(chan error, 128) + var wg sync.WaitGroup + for worker := 0; worker < 16; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for item := range jobs { + result, err := process(item) + if err != nil { + errs <- fmt.Errorf("%s %s %s: %w", item.Date, item.Vehicle.Plate, item.Vehicle.VIN, err) + continue + } + if result.RawFrameCount > 0 { + results <- result + } + } + }() + } + go func() { + for day := start; !day.After(end); day = day.AddDate(0, 0, 1) { + for _, vehicle := range vehicles { + if len(strings.TrimSpace(vehicle.VIN)) != 17 || vehicle.TankCapacityL <= 0 || vehicle.BatteryCapacityKWh <= 0 { + continue + } + jobs <- job{Vehicle: vehicle, Date: day.Format("2006-01-02")} + } + } + close(jobs) + wg.Wait() + close(results) + close(errs) + }() + + var rows []row + completed := 0 + for result := range results { + rows = append(rows, result) + completed++ + if completed%25 == 0 { + fmt.Fprintf(os.Stderr, "completed=%d latest=%s %s quality=%s raw=%d\n", completed, result.Date, result.Plate, result.CurrentQuality, result.RawFrameCount) + } + } + var errorMessages []string + for err := range errs { + errorMessages = append(errorMessages, err.Error()) + } + if len(errorMessages) > 0 { + panic(strings.Join(errorMessages, "\n")) + } + sort.Slice(rows, func(i, j int) bool { + if rows[i].Date != rows[j].Date { + return rows[i].Date < rows[j].Date + } + if rows[i].Plate != rows[j].Plate { + return rows[i].Plate < rows[j].Plate + } + return rows[i].VIN < rows[j].VIN + }) + writeJSON(filepath.Join(outputDir, "results.json"), rows) + writeCSV(filepath.Join(outputDir, "results.csv"), rows) + writeJSON(filepath.Join(outputDir, "summary.json"), summarize(rows)) + fmt.Printf("vehicle_days=%d output=%s\n", len(rows), outputDir) +} + +func process(item job) (row, error) { + frames, total, err := fetchFrames(item.Vehicle.VIN, item.Date) + if err != nil || total == 0 { + return row{Date: item.Date, Plate: item.Vehicle.Plate, VIN: item.Vehicle.VIN, Model: item.Vehicle.Model, RawFrameCount: total}, err + } + sort.SliceStable(frames, func(i, j int) bool { + if frames[i].EventTime != frames[j].EventTime { + return frames[i].EventTime < frames[j].EventTime + } + return frames[i].EventID < frames[j].EventID + }) + observations := make([]openplatform.HydrogenObservation, 0, len(frames)) + energySamples := make([]energySample, 0, len(frames)) + for _, frame := range frames { + if observation, ok := observationFromFrame(frame, item.Vehicle.TankCapacityL, item.Date); ok { + observations = append(observations, observation) + } + if sample, ok := energySampleFromFrame(frame, item.Date); ok { + energySamples = append(energySamples, sample) + } + } + params := map[string]openplatform.HydrogenCalculationParameters{ + item.Vehicle.VIN: {BatteryCapacityKWh: item.Vehicle.BatteryCapacityKWh, HydrogenEnergyKWhKg: hydrogenEnergyKWh}, + } + stats := openplatform.BuildHydrogenDailyStatsOrderedWithParameters(observations, item.Date, 0.05, 20, params) + stat := openplatform.HydrogenDailyStat{VIN: item.Vehicle.VIN, Date: item.Date, QualityStatus: "NO_DATA", QualityReason: "无有效压力温度样本"} + if len(stats) == 1 { + stat = stats[0] + } + energy := calculateEnergy(energySamples, item.Vehicle.BatteryCapacityKWh) + totalMileage := stat.MixedMileageKm + stat.PureElectricMileageKm + result := row{ + Date: item.Date, Plate: item.Vehicle.Plate, VIN: item.Vehicle.VIN, Model: item.Vehicle.Model, + RawFrameCount: total, CurrentQuality: stat.QualityStatus, CurrentReason: stat.QualityReason, + RefuelCount: stat.RefuelCount, ChargeCount: stat.ChargeCount, + TotalMileageKm: totalMileage, PureElectricMileageKm: stat.PureElectricMileageKm, + CurrentMixedMileageKm: stat.MixedMileageKm, PhysicalHydrogenKg: stat.ConsumptionKg, + FullDayStartSOC: energy.StartSOC, FullDayEndSOC: energy.EndSOC, + FullDaySOCChangePct: energy.SOCChangePctEndMinusStart, + StoredEnergyChangeKWhSOC: energy.StoredEnergyChangeKWhSOC, + BatteryNetOutputKWhIntegrated: energy.BatteryNetOutputKWhIntegrated, + FuelCellOutputKWhIntegrated: energy.FuelCellOutputKWhIntegrated, + BatteryCoverageRatio: energy.BatteryCoverageRatio, + FuelCellCoverageRatio: energy.FuelCellCoverageRatio, + ExternalChargeFrameCount: energy.ExternalChargeFrameCount, + } + if stat.SOCBalancedConsumptionKg != nil { + result.CurrentSOCBalancedKg = *stat.SOCBalancedConsumptionKg + } + if stat.SOCBalancedKgPer100Km != nil { + result.CurrentRatePerMixedKm = *stat.SOCBalancedKgPer100Km + } + if totalMileage > 0 { + result.PhysicalRatePerTotalKm = round(stat.ConsumptionKg * 100 / totalMileage) + result.BatteryEquivalentKgFixed16 = round(-energy.StoredEnergyChangeKWhSOC / hydrogenEnergyKWh) + result.StandardLikeBalancedKg = round(stat.ConsumptionKg + result.BatteryEquivalentKgFixed16) + result.StandardLikeRatePerTotalKm = round(result.StandardLikeBalancedKg * 100 / totalMileage) + switch { + case energy.ExternalChargeFrameCount > 0: + result.StandardLikeReason = "检测到外部充电，整日SOC平衡修正不适用，应按CD/CS分段" + case result.StandardLikeBalancedKg < 0: + result.StandardLikeReason = "SOC平衡氢量为负，端点、SOC或换算系数需复核" + default: + result.StandardLikeApplicable = true + } + } else { + result.StandardLikeReason = "无有效总里程" + } + // The GB/T 43252 contribution trial uses positive battery net output and + // fuel-cell output measured over non-external-charge vehicle-on intervals. + // Coverage gates prevent sparse power fields from being treated as precise. + if totalMileage >= 10 && stat.ConsumptionKg > 0 && energy.ExternalChargeFrameCount == 0 && energy.BatteryCoverageRatio >= 0.98 && energy.FuelCellCoverageRatio >= 0.98 && energy.FuelCellOutputKWhIntegrated > 0 { + batteryOutput := math.Max(0, energy.BatteryNetOutputKWhIntegrated) + totalOutput := batteryOutput + energy.FuelCellOutputKWhIntegrated + if totalOutput > 0 { + result.PowerIntegrationUsable = true + result.BatteryEnergyShare = round(batteryOutput / totalOutput) + result.FuelCellEnergyShare = round(energy.FuelCellOutputKWhIntegrated / totalOutput) + result.BatteryContributionKm = round(totalMileage * result.BatteryEnergyShare) + result.FuelCellContributionKm = round(totalMileage * result.FuelCellEnergyShare) + if result.FuelCellContributionKm > 0 { + result.HydrogenRatePerFCContributionKm = round(stat.ConsumptionKg * 100 / result.FuelCellContributionKm) + } + } + } + return result, nil +} + +func calculateEnergy(samples []energySample, batteryCapacityKWh float64) energyResult { + result := energyResult{EnergySampleCount: len(samples)} + if len(samples) == 0 { + return result + } + for _, sample := range samples { + if sample.HasChargeStatus && sample.ChargeStatus == 1 { + result.ExternalChargeFrameCount++ + } + } + operating := make([]energySample, 0, len(samples)) + for _, sample := range samples { + if sample.HasVehicleStatus && sample.VehicleStatus == 1 && (!sample.HasChargeStatus || sample.ChargeStatus != 1) { + operating = append(operating, sample) + } + } + if len(operating) == 0 { + return result + } + result.FirstTime = operating[0].At.Format(time.RFC3339) + result.LastTime = operating[len(operating)-1].At.Format(time.RFC3339) + result.OperatingSpanSeconds = operating[len(operating)-1].At.Sub(operating[0].At).Seconds() + for _, sample := range operating { + if sample.HasSOC { + result.StartSOC = sample.SOC + break + } + } + for index := len(operating) - 1; index >= 0; index-- { + if operating[index].HasSOC { + result.EndSOC = operating[index].SOC + break + } + } + result.SOCChangePctEndMinusStart = round(result.EndSOC - result.StartSOC) + result.StoredEnergyChangeKWhSOC = round(batteryCapacityKWh * result.SOCChangePctEndMinusStart / 100) + result.BatteryNetOutputKWhSOC = round(-result.StoredEnergyChangeKWhSOC) + for index := 1; index < len(operating); index++ { + previous, current := operating[index-1], operating[index] + delta := current.At.Sub(previous.At) + if delta <= 0 || delta > maxIntegrationGap { + continue + } + hours := delta.Hours() + if previous.HasBatteryPower && current.HasBatteryPower { + power0 := previous.BatteryVoltageV * previous.BatteryCurrentA / 1000 + power1 := current.BatteryVoltageV * current.BatteryCurrentA / 1000 + result.BatteryNetOutputKWhIntegrated += (power0 + power1) * 0.5 * hours + result.BatteryCoverageSeconds += delta.Seconds() + } + if previous.HasFuelCellPower && current.HasFuelCellPower { + power0 := previous.FuelCellVoltageV * previous.FuelCellCurrentA / 1000 + power1 := current.FuelCellVoltageV * current.FuelCellCurrentA / 1000 + result.FuelCellOutputKWhIntegrated += (power0 + power1) * 0.5 * hours + result.FuelCellCoverageSeconds += delta.Seconds() + } + } + result.BatteryNetOutputKWhIntegrated = round(result.BatteryNetOutputKWhIntegrated) + result.FuelCellOutputKWhIntegrated = round(result.FuelCellOutputKWhIntegrated) + if result.OperatingSpanSeconds > 0 { + result.BatteryCoverageRatio = round(result.BatteryCoverageSeconds / result.OperatingSpanSeconds) + result.FuelCellCoverageRatio = round(result.FuelCellCoverageSeconds / result.OperatingSpanSeconds) + } + return result +} + +func observationFromFrame(frame rawFrame, tankCapacity float64, statDate string) (openplatform.HydrogenObservation, bool) { + if frame.MessageID != 2 || !strings.EqualFold(frame.ParseStatus, "OK") { + return openplatform.HydrogenObservation{}, false + } + pressure, pressureOK := number(frame.ParsedFields["gb32960.fuel_cell.max_hydrogen_pressure_mpa"]) + temperature, temperatureOK := number(frame.ParsedFields["gb32960.fuel_cell.max_hydrogen_temperature_c"]) + if !pressureOK || !temperatureOK || pressure <= 0 || pressure > 70 || temperature <= -40 || temperature > 726.85 { + return openplatform.HydrogenObservation{}, false + } + mass, ok := openplatform.PressureHydrogenMassKg(pressure, temperature, tankCapacity) + if !ok { + return openplatform.HydrogenObservation{}, false + } + step, _ := openplatform.PressureHydrogenMassKg(math.Max(0, pressure-0.2), temperature, tankCapacity) + at, err := parseEventTime(frame.EventTime) + if err != nil || at.Format("2006-01-02") != statDate { + return openplatform.HydrogenObservation{}, false + } + parsed, _ := json.Marshal(frame.ParsedFields) + _, _, active, known, _ := openplatform.ExtractHydrogenTelemetry(string(parsed)) + observation := openplatform.HydrogenObservation{ + VIN: frame.VIN, Source: frame.SourceEndpoint, EventID: frame.EventID, ObservedAt: at, + MassKg: mass, TankCapacityLiter: tankCapacity, PressureMPa: pressure, TemperatureC: temperature, + NoiseKg: math.Min(1, math.Max(0.05, mass-step)), RefuelThresholdKg: math.Max(1, mass*0.05), + FuelCellActive: active, FuelCellStateKnown: known, + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.soc_percent"]); ok && value >= 0 && value <= 100 { + observation.SOCPercent, observation.SOCKnown = value, true + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.total_mileage_km"]); ok && value >= 0 { + observation.MileageKm, observation.MileageKnown = value, true + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.vehicle_status"]); ok { + observation.VehicleState, observation.VehicleStateKnown = int(value), value >= 0 && value <= 255 + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.charge_status"]); ok { + observation.ChargeState, observation.ChargeStateKnown = int(value), value >= 0 && value <= 255 + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.running_mode"]); ok { + observation.RunningMode, observation.RunningModeKnown = int(value), value >= 0 && value <= 255 + } + return observation, true +} + +func energySampleFromFrame(frame rawFrame, statDate string) (energySample, bool) { + if frame.MessageID != 2 || !strings.EqualFold(frame.ParseStatus, "OK") { + return energySample{}, false + } + at, err := parseEventTime(frame.EventTime) + if err != nil || at.Format("2006-01-02") != statDate { + return energySample{}, false + } + sample := energySample{At: at} + if value, ok := number(frame.ParsedFields["gb32960.vehicle.vehicle_status"]); ok { + sample.VehicleStatus, sample.HasVehicleStatus = int(value), true + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.charge_status"]); ok { + sample.ChargeStatus, sample.HasChargeStatus = int(value), true + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.soc_percent"]); ok && value >= 0 && value <= 100 { + sample.SOC, sample.HasSOC = value, true + } + if value, ok := number(frame.ParsedFields["gb32960.vehicle.total_mileage_km"]); ok && value >= 0 { + sample.Mileage, sample.HasMileage = value, true + } + batteryVoltage, batteryVoltageOK := number(frame.ParsedFields["gb32960.vehicle.total_voltage_v"]) + batteryCurrent, batteryCurrentOK := number(frame.ParsedFields["gb32960.vehicle.total_current_a"]) + if batteryVoltageOK && batteryCurrentOK && batteryVoltage > 0 && batteryVoltage <= 1000 && batteryCurrent >= -1000 && batteryCurrent <= 1000 { + sample.BatteryVoltageV, sample.BatteryCurrentA, sample.HasBatteryPower = batteryVoltage, batteryCurrent, true + } + fuelCellVoltage, fuelCellVoltageOK := number(frame.ParsedFields["gb32960.fuel_cell.fuel_cell_voltage_v"]) + fuelCellCurrent, fuelCellCurrentOK := number(frame.ParsedFields["gb32960.fuel_cell.fuel_cell_current_a"]) + if fuelCellVoltageOK && fuelCellCurrentOK && fuelCellVoltage >= 0 && fuelCellVoltage <= 2000 && fuelCellCurrent >= 0 && fuelCellCurrent <= 2000 { + sample.FuelCellVoltageV, sample.FuelCellCurrentA, sample.HasFuelCellPower = fuelCellVoltage, fuelCellCurrent, true + } + return sample, sample.HasVehicleStatus || sample.HasSOC || sample.HasBatteryPower || sample.HasFuelCellPower +} + +func fetchFrames(vin, date string) ([]rawFrame, int, error) { + const limit = 500 + first, err := fetchPage(vin, date, 0, limit, true) + if err != nil { + return nil, 0, err + } + frames := append([]rawFrame(nil), first.Items...) + for offset := limit; offset < first.Total; offset += limit { + page, err := fetchPage(vin, date, offset, limit, false) + if err != nil { + return nil, first.Total, err + } + frames = append(frames, page.Items...) + } + if len(frames) != first.Total { + return nil, first.Total, fmt.Errorf("API total=%d fetched=%d", first.Total, len(frames)) + } + return frames, first.Total, nil +} + +func fetchPage(vin, date string, offset, limit int, includeTotal bool) (rawResponse, error) { + query := url.Values{ + "protocol": {"GB32960"}, "vin": {vin}, + "dateFrom": {date + " 00:00:00"}, "dateTo": {date + " 23:59:59"}, + "orderBy": {"eventTime"}, "limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(offset)}, + "includeFields": {"true"}, "includePayload": {"false"}, "includeTotal": {strconv.FormatBool(includeTotal)}, + } + var response *http.Response + var err error + for attempt := 1; attempt <= 4; attempt++ { + request, _ := http.NewRequest(http.MethodGet, baseURL+"/api/history/raw-frames?"+query.Encode(), nil) + request.Header.Set("Accept-Encoding", "gzip") + response, err = (&http.Client{Timeout: 90 * time.Second}).Do(request) + if err == nil { + break + } + if attempt < 4 { + time.Sleep(time.Duration(attempt) * 500 * time.Millisecond) + } + } + if err != nil { + return rawResponse{}, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + body, _ := io.ReadAll(response.Body) + if response.StatusCode == http.StatusInternalServerError && bytes.Contains(body, []byte("Table does not exist")) { + return rawResponse{}, nil + } + return rawResponse{}, fmt.Errorf("HTTP %d: %s", response.StatusCode, body) + } + var reader io.Reader = response.Body + if response.Header.Get("Content-Encoding") == "gzip" { + gzipReader, err := gzip.NewReader(response.Body) + if err != nil { + return rawResponse{}, err + } + defer gzipReader.Close() + reader = gzipReader + } + var result rawResponse + if err := json.NewDecoder(reader).Decode(&result); err != nil { + return rawResponse{}, err + } + return result, nil +} + +func summarize(rows []row) map[string]any { + byDate := map[string]map[string]any{} + for _, date := range uniqueDates(rows) { + var dateRows []row + for _, value := range rows { + if value.Date == date { + dateRows = append(dateRows, value) + } + } + quality := map[string]int{} + valid := 0 + standardApplicable := 0 + powerUsable := 0 + physicalTotal, balancedTotal, mileageTotal, balancedMileageTotal := 0.0, 0.0, 0.0, 0.0 + for _, value := range dateRows { + quality[value.CurrentQuality]++ + if value.CurrentQuality != "NO_DATA" && value.TotalMileageKm > 0 { + valid++ + physicalTotal += value.PhysicalHydrogenKg + mileageTotal += value.TotalMileageKm + } + if value.StandardLikeApplicable { + standardApplicable++ + balancedTotal += value.StandardLikeBalancedKg + balancedMileageTotal += value.TotalMileageKm + } + if value.PowerIntegrationUsable { + powerUsable++ + } + } + physicalRate, balancedRate := 0.0, 0.0 + if mileageTotal > 0 { + physicalRate = round(physicalTotal * 100 / mileageTotal) + } + if balancedMileageTotal > 0 { + balancedRate = round(balancedTotal * 100 / balancedMileageTotal) + } + byDate[date] = map[string]any{ + "vehicleDaysWithFrames": len(dateRows), "validVehicleDays": valid, "standardLikeApplicableVehicleDays": standardApplicable, "qualityCounts": quality, + "powerIntegrationUsableVehicleDays": powerUsable, "totalMileageKm": round(mileageTotal), + "physicalHydrogenKg": round(physicalTotal), "standardLikeBalancedHydrogenKg": round(balancedTotal), + "standardLikeMileageKm": round(balancedMileageTotal), "fleetPhysicalRateKgPer100Km": physicalRate, "fleetStandardLikeRateKgPer100Km": balancedRate, + } + } + return map[string]any{"vehicleDays": len(rows), "byDate": byDate} +} + +func writeCSV(path string, rows []row) { + file, err := os.Create(path) + if err != nil { + panic(err) + } + defer file.Close() + writer := csv.NewWriter(file) + defer writer.Flush() + headers := []string{"日期", "车牌", "VIN", "车型", "原始帧数", "当前质量状态", "当前质量原因", "加氢次数", "充电次数", "总里程km", "当前纯电里程km", "当前混动里程km", "物理耗氢kg", "当前SOC平衡氢量kg", "当前修正氢耗_按混动里程", "物理氢耗_按总里程", "全日起始SOC", "全日终止SOC", "全日SOC变化_末减初", "全日电池储能变化kWh", "电池等效氢量_16kWhkg", "标准式平衡氢量kg", "标准式平衡氢耗_按总里程", "标准式修正可用", "标准式修正说明", "电流积分电池净输出kWh", "燃料电池输出积分kWh", "电池能量贡献占比", "燃料电池能量贡献占比", "电池贡献里程km", "燃料电池贡献里程km", "按燃料电池贡献里程氢耗", "电池功率覆盖率", "燃料电池功率覆盖率", "外充状态帧数", "功率积分可用"} + _ = writer.Write(headers) + for _, value := range rows { + _ = writer.Write([]string{ + value.Date, value.Plate, value.VIN, value.Model, integer(value.RawFrameCount), value.CurrentQuality, value.CurrentReason, + integer(value.RefuelCount), integer(value.ChargeCount), decimal(value.TotalMileageKm), decimal(value.PureElectricMileageKm), + decimal(value.CurrentMixedMileageKm), decimal(value.PhysicalHydrogenKg), decimal(value.CurrentSOCBalancedKg), + decimal(value.CurrentRatePerMixedKm), decimal(value.PhysicalRatePerTotalKm), decimal(value.FullDayStartSOC), + decimal(value.FullDayEndSOC), decimal(value.FullDaySOCChangePct), decimal(value.StoredEnergyChangeKWhSOC), + decimal(value.BatteryEquivalentKgFixed16), decimal(value.StandardLikeBalancedKg), decimal(value.StandardLikeRatePerTotalKm), + strconv.FormatBool(value.StandardLikeApplicable), value.StandardLikeReason, + decimal(value.BatteryNetOutputKWhIntegrated), decimal(value.FuelCellOutputKWhIntegrated), decimal(value.BatteryEnergyShare), + decimal(value.FuelCellEnergyShare), decimal(value.BatteryContributionKm), decimal(value.FuelCellContributionKm), + decimal(value.HydrogenRatePerFCContributionKm), decimal(value.BatteryCoverageRatio), decimal(value.FuelCellCoverageRatio), + integer(value.ExternalChargeFrameCount), strconv.FormatBool(value.PowerIntegrationUsable), + }) + } +} + +func uniqueDates(rows []row) []string { + seen := map[string]bool{} + for _, value := range rows { + seen[value.Date] = true + } + dates := make([]string, 0, len(seen)) + for date := range seen { + dates = append(dates, date) + } + sort.Strings(dates) + return dates +} + +func number(value any) (float64, bool) { + switch typed := value.(type) { + case float64: + return typed, true + case json.Number: + parsed, err := typed.Float64() + return parsed, err == nil + case string: + parsed, err := strconv.ParseFloat(strings.TrimSpace(typed), 64) + return parsed, err == nil + default: + return 0, false + } +} + +func parseEventTime(value string) (time.Time, error) { + location := time.FixedZone("CST", 8*3600) + if parsed, err := time.ParseInLocation("2006-01-02 15:04:05.000", value, location); err == nil { + return parsed, nil + } + return time.ParseInLocation("2006-01-02 15:04:05", value, location) +} + +func mustDate(value string) time.Time { + parsed, err := time.Parse("2006-01-02", value) + if err != nil { + panic(err) + } + return parsed +} + +func mustReadJSON(path string, target any) { + data, err := os.ReadFile(path) + if err != nil { + panic(err) + } + if err := json.Unmarshal(data, target); err != nil { + panic(err) + } +} + +func writeJSON(path string, value any) { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + panic(err) + } + if err := os.WriteFile(path, data, 0o644); err != nil { + panic(err) + } +} + +func round(value float64) float64 { return math.Round(value*1000) / 1000 } +func decimal(value float64) string { return strconv.FormatFloat(value, 'f', 3, 64) } +func integer(value int) string { return strconv.Itoa(value) } diff --git a/vehicle-data-platform/apps/web/src/api/client.ts b/vehicle-data-platform/apps/web/src/api/client.ts index d95217d3..43498821 100644 --- a/vehicle-data-platform/apps/web/src/api/client.ts +++ b/vehicle-data-platform/apps/web/src/api/client.ts @@ -477,7 +477,7 @@ export const api = { }), vehicleRealtime: (params = new URLSearchParams(), signal?: AbortSignal) => request>(`/api/realtime/vehicles?${params.toString()}`, signal ? { signal } : undefined), realtimeLocations: (params = new URLSearchParams()) => request>(`/api/realtime/locations?${params.toString()}`), - historyLocations: (params = new URLSearchParams()) => request>(`/api/history/locations?${params.toString()}`), + historyLocations: (params = new URLSearchParams(), signal?: AbortSignal) => request>(`/api/history/locations?${params.toString()}`, withSignal(undefined, signal)), rawFrames: (params = new URLSearchParams()) => request>(`/api/history/raw-frames?${params.toString()}`), rawFramesQuery: (query: RawFrameQuery) => request>('/api/history/raw-frames/query', { method: 'POST', diff --git a/vehicle-data-platform/apps/web/src/api/types.ts b/vehicle-data-platform/apps/web/src/api/types.ts index fd0cb004..aa416ee4 100644 --- a/vehicle-data-platform/apps/web/src/api/types.ts +++ b/vehicle-data-platform/apps/web/src/api/types.ts @@ -381,6 +381,8 @@ export interface AlertQuery { keyword?: string; severity?: string; status?: stri export interface AlertSummary { active: number; unprocessed: number; processing: number; recovered: number; closed: number; ignored: number; unreadNotifications: number; asOf: string; } export interface AlertAction { id: number; action: string; fromStatus: string; toStatus: string; actor: string; note: string; createdAt: string; } export interface AlertEvent { + nativeAlarmFields?: Record; + nativeAlarmNames?: string[]; nativeAlarmReservedBits?: number[]; id: string; eventType?: string; eventCategory?: string; executionState?: 'pending' | 'processing' | 'recovered' | 'completed' | 'ignored'; ruleId: string; ruleName: string; ruleVersion: number; severity: AlertSeverity; triggerType?: AlertTriggerType; status: AlertStatus; vin: string; plate: string; protocol: string; metric: string; operator: string; triggerValue: number; threshold: number; thresholdHigh: number; unit: string; durationSec: number; location: string; longitude?: number; latitude?: number; sourceEventId: string; @@ -901,6 +903,12 @@ export interface LatestTelemetryResponse { } export interface DailyMileageRow { + province?: string; + city?: string; + region?: string; + locationTime?: string; + locationStatus?: string; + hydrogenPhysicalConsumptionKg?: number | null; vin: string; plate: string; date: string; @@ -941,7 +949,7 @@ export interface HydrogenDailyEvidence { vin: string; plate: string; date: string; source: string; rawConsumptionKg: number; batterySocDeltaPct?: number; batteryDischargeKWh?: number; batteryEquivalentKg?: number; socBalancedConsumptionKg?: number; mixedMileageKm: number; pureElectricMileageKm: number; - consumptionKgPer100Km?: number; socBalancedKgPer100Km?: number; sampleCount: number; + physicalConsumptionKgPer100Km?: number; consumptionKgPer100Km?: number; socBalancedKgPer100Km?: number; sampleCount: number; refuelCount: number; refuelAmountKg?: number; chargeCount: number; chargeEnergyKWh?: number; validSegmentCount: number; invalidSegmentCount: number; qualityStatus: string; qualityReason: string; algorithmVersion: string; calculatedAt: string; parameters: { diff --git a/vehicle-data-platform/apps/web/src/v2/domain/alert.ts b/vehicle-data-platform/apps/web/src/v2/domain/alert.ts index eeece2e0..5666a02a 100644 --- a/vehicle-data-platform/apps/web/src/v2/domain/alert.ts +++ b/vehicle-data-platform/apps/web/src/v2/domain/alert.ts @@ -14,7 +14,7 @@ const LEGACY_LOCAL_SQL_TIMESTAMP = /\.\d{6}Z$/; export const severityLabels: Record = { critical: '紧急', major: '重要', minor: '一般' }; export const statusLabels: Record = { unprocessed: '未处理', processing: '处理中', recovered: '已恢复', closed: '已关闭', ignored: '已忽略' }; -export const actionLabels: Record = { trigger: '触发', acknowledge: '已确认', close: '已关闭', ignore: '已忽略', recover: '已恢复', processing: '处理中', recovered: '已恢复', closed: '已关闭', ignored: '已忽略' }; +export const actionLabels: Record = { repair: '修正告警信息', trigger: '触发', acknowledge: '已确认', close: '已关闭', ignore: '已忽略', recover: '已恢复', processing: '处理中', recovered: '已恢复', closed: '已关闭', ignored: '已忽略' }; export const metricLabels: Record = { speed_kmh: '速度', soc_percent: 'SOC', alarm_active: '协议告警位', freshness_sec: '离线时长', data_delay_sec: '数据延迟' }; export const operatorLabels: Record = { gt: '>', gte: '≥', lt: '<', lte: '≤', eq: '=', neq: '≠', between: '区间内', outside: '区间外', changed: '状态变化' }; export const triggerTypeLabels: Record = { metric: '数值触发', geofence: '电子围栏', stationary: '长时间静止', offline: '长时间离线' }; diff --git a/vehicle-data-platform/apps/web/src/v2/domain/event.ts b/vehicle-data-platform/apps/web/src/v2/domain/event.ts index 1f481c7d..43eb106c 100644 --- a/vehicle-data-platform/apps/web/src/v2/domain/event.ts +++ b/vehicle-data-platform/apps/web/src/v2/domain/event.ts @@ -145,3 +145,25 @@ export function automationSource(rule: Pick) { if (protocols.length === 1) return protocolEventSources.find((item) => item.protocol === protocols[0])?.label ?? protocols[0]; return `${protocols.length} 类协议`; } + +export function isNativeAlarm(event: Pick) { + return event.ruleId === 'native-gb32960-alarm'; +} + +export function nativeAlarmDetails(event: AlertEvent) { + const fields = event.nativeAlarmFields; + if (!fields) return []; + return [ + ['max_alarm_level', '最高报警等级'], + ['general_alarm_flag', '通用报警标志'], + ['battery_faults', '可充电储能装置故障码'], + ['motor_faults', '驱动电机故障码'], + ['engine_faults', '发动机故障码'], + ['other_faults', '其他故障码'], + ].map(([name, label]) => { + const value = fields[`gb32960.alarm.${name}`]; + let text = value == null ? '未上报' : Array.isArray(value) ? (value.length ? value.join('、') : '无') : String(value); + if (name === 'max_alarm_level' && value != null) text = ({ '0': '0 · 无故障', '1': '1 · 一级故障', '2': '2 · 二级故障', '3': '3 · 三级故障', '254': '254 · 异常', '255': '255 · 无效' } as Record)[String(value)] ?? text; + return { key: label, value: text }; + }); +} diff --git a/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.test.ts b/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.test.ts index b36c087f..2f8da2ea 100644 --- a/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.test.ts +++ b/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.test.ts @@ -14,7 +14,7 @@ test('creates a styled numeric mileage workbook with formulas and frozen panes', { vin: 'LTEST000000000002', plate: '粤A54321', brandName: '飞驰', modelName: 'FSQ' } ], mileageRows: [ - { vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', startMileageKm: 100, endMileageKm: 188.7, dailyMileageKm: 88.7, pureHydrogenMileageKm: 56.2, pureElectricMileageKm: 32.5, mixedMileageKm: 56.2, hydrogenConsumptionKg: 3.1, hydrogenConsumptionKgPer100Km: 5.5, hydrogenSocBalancedKg: 3.126, batterySocDeltaPct: -2, chargeCount: 1, chargeEnergyKWh: 12.5, refuelCount: 1, refuelAmountKg: 8.976, hydrogenQualityStatus: 'SUSPECT', hydrogenQualityReason: '疑似管路泄压', source: 'GB32960' }, + { vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', province: '广东省', city: '广州市', region: '华南', locationTime: '2026-07-13 23:00:00', locationStatus: '已解析', startMileageKm: 100, endMileageKm: 188.7, dailyMileageKm: 88.7, pureHydrogenMileageKm: 56.2, pureElectricMileageKm: 32.5, mixedMileageKm: 56.2, hydrogenConsumptionKg: 3.126, hydrogenPhysicalConsumptionKg: 3.1, hydrogenConsumptionKgPer100Km: 5.5, hydrogenSocBalancedKg: 3.126, batterySocDeltaPct: -2, chargeCount: 1, chargeEnergyKWh: 12.5, refuelCount: 1, refuelAmountKg: 8.976, hydrogenQualityStatus: 'SUSPECT', hydrogenQualityReason: '疑似管路泄压', source: 'GB32960' }, { vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-14', startMileageKm: 188.7, endMileageKm: 293.3, dailyMileageKm: 104.6, pureHydrogenMileageKm: 72.2, hydrogenConsumptionKg: 4.2, hydrogenConsumptionKgPer100Km: 5.8, source: 'GB32960' } ], sources: [ @@ -43,12 +43,12 @@ test('creates a styled numeric mileage workbook with formulas and frozen panes', expect((sheet as unknown as { conditionalFormattings: unknown[] }).conditionalFormattings).toHaveLength(1); const hydrogenSheet = workbook.getWorksheet('氢能明细')!; expect(hydrogenSheet.getRow(1).values).toEqual([ - undefined, '日期', '车牌', 'VIN', '品牌', '车型', '每日氢耗 (kg/100km)', '结果状态', '总里程 (km)', '纯电里程 (km)', '混动里程 (km)', '电SOC差值 (百分点)', '充电次数', '充电量 (kWh)', '加氢次数', '加氢量 (kg)', '修正用氢量 (kg)', '原因', '物理用氢量 (kg)', '来源' + undefined, '日期', '车牌', '当天所在省', '当天所在市', '大区', 'VIN', '品牌', '车型', '每日氢耗 (kg/100km)', '结果状态', '总里程 (km)', '纯电里程 (km)', '混动里程 (km)', '电SOC差值 (百分点)', '充电次数', '充电量 (kWh)', '加氢次数', '加氢量 (kg)', '修正用氢量 (kg)', '原因', '物理用氢量 (kg)', '来源', '定位时间（当日最后有效点）', '位置状态' ]); expect(hydrogenSheet.getRow(2).values).toEqual([ - undefined, '2026-07-13', '粤A12345', 'LTEST000000000001', '现代', 'XCIENT', 5.5, 'SUSPECT', 88.7, 32.5, 56.2, -2, 1, 12.5, 1, 8.976, 3.126, '疑似管路泄压', 3.1, 'GB32960' + undefined, '2026-07-13', '粤A12345', '广东省', '广州市', '华南', 'LTEST000000000001', '现代', 'XCIENT', 5.5, 'SUSPECT', 88.7, 32.5, 56.2, -2, 1, 12.5, 1, 8.976, 3.126, '疑似管路泄压', 3.1, 'GB32960', '2026-07-13 23:00:00', '已解析' ]); - expect(hydrogenSheet.autoFilter).toEqual({ from: { row: 1, column: 1 }, to: { row: 3, column: 19 } }); + expect(hydrogenSheet.autoFilter).toEqual({ from: { row: 1, column: 1 }, to: { row: 3, column: 24 } }); expect((await workbook.xlsx.writeBuffer()).byteLength).toBeGreaterThan(5_000); }); @@ -93,18 +93,20 @@ test('terminates the workbook worker when an active export is cancelled', async dateTo: '2026-07-14', dates: ['2026-07-13', '2026-07-14'], vehicles: [], - mileageRows: [{ vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', startMileageKm: 100, endMileageKm: 188.7, dailyMileageKm: 88.7, source: 'GB32960' }], + mileageRows: [{ vin: 'LTEST000000000001', plate: '粤A12345', date: '2026-07-13', province: '广东省', city: '广州市', region: '华南', locationTime: '2026-07-13 23:00:00', locationStatus: '已解析', startMileageKm: 100, endMileageKm: 188.7, dailyMileageKm: 88.7, source: 'GB32960' }], sources: [] }, controller.signal); expect(postMessage).toHaveBeenCalledTimes(3); expect(postMessage.mock.calls.map(([message]) => message.type)).toEqual(['start', 'rows', 'finish']); expect(postMessage.mock.calls[1][0].rows).toEqual([{ + province: '广东省', city: '广州市', region: '华南', locationTime: '2026-07-13 23:00:00', locationStatus: '已解析', vin: 'LTEST000000000001', date: '2026-07-13', dailyMileageKm: 88.7, pureHydrogenMileageKm: undefined, hydrogenConsumptionKg: undefined, + hydrogenPhysicalConsumptionKg: undefined, hydrogenConsumptionKgPer100Km: undefined, hydrogenSocBalancedKg: undefined, pureElectricMileageKm: undefined, @@ -143,3 +145,10 @@ test('releases a streaming export before finish when the route aborts', async () expect(terminate).toHaveBeenCalledTimes(1); await expect(stream.finish([{ vin: 'VIN001', plate: '粤A00001' }])).rejects.toMatchObject({ name: 'AbortError' }); }); + + +test('hydrogen export opens the hydrogen sheet with geography in the first five columns', async () => { + const workbook = await createMileageWorkbook({ metricView: 'hydrogen', dateFrom: '2026-09-01', dateTo: '2026-09-01', dates: ['2026-09-01'], vehicles: [], mileageRows: [], sources: [] }); + expect(workbook.worksheets[0].name).toBe('氢能明细'); + expect(Array.from(workbook.worksheets[0].getRow(1).values as unknown[]).slice(1, 6)).toEqual(['日期', '车牌', '当天所在省', '当天所在市', '大区']); +}); diff --git a/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.ts b/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.ts index 29fb94ed..26a91393 100644 --- a/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.ts +++ b/vehicle-data-platform/apps/web/src/v2/domain/mileageExport.ts @@ -6,6 +6,7 @@ export type MileageExportSource = { protocol: string; label: string; mileageType export type MileageWorkbookRow = Pick & Partial>; export type MileageExportInput = { + metricView?: 'mileage' | 'hydrogen'; dateFrom: string; dateTo: string; dates: string[]; @@ -68,7 +69,7 @@ export function createMileageExportStream(input: Omit ({ + province: row.province, city: row.city, region: row.region, locationTime: row.locationTime, locationStatus: row.locationStatus, vin: row.vin, date: row.date, dailyMileageKm: row.dailyMileageKm, pureHydrogenMileageKm: row.pureHydrogenMileageKm, hydrogenConsumptionKg: row.hydrogenConsumptionKg, + hydrogenPhysicalConsumptionKg: row.hydrogenPhysicalConsumptionKg, hydrogenConsumptionKgPer100Km: row.hydrogenConsumptionKgPer100Km, hydrogenSocBalancedKg: row.hydrogenSocBalancedKg, pureElectricMileageKm: row.pureElectricMileageKm, @@ -126,6 +129,7 @@ export function createMileageExportStream(input: Omit { - cell.font = { name: columnNumber === 3 ? 'Consolas' : 'Microsoft YaHei', size: columnNumber === 3 ? 9 : 10, color: { argb: 'FF34445A' } }; - cell.alignment = { vertical: 'middle', horizontal: columnNumber >= 6 && columnNumber <= 16 ? 'right' : 'left' }; + cell.font = { name: columnNumber === 6 ? 'Consolas' : 'Microsoft YaHei', size: columnNumber === 6 ? 9 : 10, color: { argb: 'FF34445A' } }; + cell.alignment = { vertical: 'middle', horizontal: columnNumber >= 9 && columnNumber <= 19 ? 'right' : 'left' }; cell.fill = { type: 'pattern', pattern: 'solid', fgColor: { argb: index % 2 ? 'FFF9FBFD' : 'FFFFFFFF' } }; cell.border = { bottom: { style: 'thin', color: { argb: 'FFE6ECF3' } } }; - if ([6, 8, 9, 10, 11, 13, 15, 16, 18].includes(columnNumber)) cell.numFmt = '#,##0.000'; - if ([12, 14].includes(columnNumber)) cell.numFmt = '0'; + if ([9, 11, 12, 13, 14, 16, 18, 19, 21].includes(columnNumber)) cell.numFmt = '#,##0.000'; + if ([15, 17].includes(columnNumber)) cell.numFmt = '0'; }); }); - [13, 15, 24, 18, 24, 22, 14, 15, 17, 17, 22, 12, 17, 12, 16, 19, 52, 19, 16].forEach((width, index) => { + [13, 15, 18, 18, 12, 24, 18, 24, 22, 14, 15, 17, 17, 22, 12, 17, 12, 16, 19, 52, 19, 16, 30, 24].forEach((width, index) => { hydrogenSheet.getColumn(index + 1).width = width; }); if (energyRows.length) { diff --git a/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.test.tsx b/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.test.tsx index 494b7366..ea58c90f 100644 --- a/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.test.tsx +++ b/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.test.tsx @@ -1359,3 +1359,42 @@ test('opens a recent automation run in a focused event inbox', async () => { expect(screen.getByTestId('alert-route-state')).not.toHaveTextContent('eventId='); expect(screen.getByTestId('alert-route-state')).not.toHaveTextContent('runFromAutomationId='); }); + + +test.each([false, true])('displays native GB32960 alarms without automation (mobile=%s)', async (mobile) => { + layout.mobile = mobile; + const event: AlertEvent = { + ...alertEvent('native-event', 'NATIVEVIN', '粤A原生01'), + ruleId: 'native-gb32960-alarm', ruleName: '绝缘报警、驱动电机温度报警', ruleVersion: 0, + protocol: 'GB32960', metric: 'alarm_active', + nativeAlarmNames: ['绝缘报警', '驱动电机温度报警'], nativeAlarmReservedBits: [20, 21], + nativeAlarmFields: { + 'gb32960.alarm.max_alarm_level': 2, + 'gb32960.alarm.general_alarm_flag': '0x00000004', + 'gb32960.alarm.battery_faults': ['0x00000001'], + 'gb32960.alarm.motor_faults': [], + 'gb32960.alarm.engine_faults': ['0x00000002'], + 'gb32960.alarm.other_faults': [], + }, + }; + mocks.alertSummaryV2.mockResolvedValue({ active: 1, unprocessed: 1, processing: 0, recovered: 0, closed: 0, ignored: 0, unreadNotifications: 0, asOf: '' }); + mocks.alertEventsV2.mockResolvedValue({ items: [event], total: 1, limit: 20, offset: 0 }); + mocks.alertEventV2.mockResolvedValue(event); + mocks.alertRulesV2.mockResolvedValue([]); + mocks.alertNotificationsV2.mockResolvedValue({ items: [], total: 0, limit: 100, offset: 0 }); + const client = new QueryClient({ defaultOptions: { queries: { retry: false } } }); + render(); + const fields = await screen.findByRole('region', { name: '32960 原生告警字段' }); + await waitFor(() => expect(fields).toHaveTextContent('2 · 二级故障')); + expect(fields).toHaveTextContent('0x00000004'); + expect(fields).toHaveTextContent('可充电储能装置故障码0x00000001'); + expect(fields).toHaveTextContent('驱动电机故障码无'); + expect(fields).toHaveTextContent('发动机故障码0x00000002'); + expect(fields).toHaveTextContent('其他故障码无'); + expect(screen.queryByLabelText('事件证据')).not.toBeInTheDocument(); + expect(screen.getByLabelText('具体告警信息')).toHaveTextContent('绝缘报警'); + expect(screen.getByLabelText('具体告警信息')).toHaveTextContent('驱动电机温度报警'); + expect(screen.getByLabelText('具体告警信息')).toHaveTextContent('bit20、bit21'); + expect(screen.getByLabelText('事件执行轨迹')).toHaveTextContent('车辆原生告警入库'); + expect(screen.getByLabelText('事件执行轨迹')).not.toHaveTextContent('匹配自动化'); +}); diff --git a/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.tsx b/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.tsx index a2ed61ba..99a17109 100644 --- a/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.tsx +++ b/vehicle-data-platform/apps/web/src/v2/pages/AlertsPage.tsx @@ -6,7 +6,7 @@ import { Link, useSearchParams } from 'react-router-dom'; import { api } from '../../api/client'; import type { AlertEvent, AlertNotification, AlertNotificationChannelCapability, AlertNotificationConfig, AlertNotificationRetryResult, AlertNotificationTarget, AlertQuery, AlertRule, AlertRuleInput, AlertRulePage, AlertRuleRevision, AlertStatus, AlertTriggerType, MetricDefinition, Page } from '../../api/types'; import { actionLabels, alertDeltaText, alertValue, canAct, formatAlertDuration, formatAlertTime, operatorLabels, ruleCondition, thresholdText, triggerTypeLabels } from '../domain/alert'; -import { automationEventType, automationSource, eventContract, eventEvidenceHistoryPath, protocolEventSources, toVehicleEvent } from '../domain/event'; +import { automationEventType, automationSource, eventContract, eventEvidenceHistoryPath, isNativeAlarm, nativeAlarmDetails, protocolEventSources, toVehicleEvent } from '../domain/event'; import { InlineError, PanelEmpty, PanelError, PanelLoading } from '../shared/AsyncState'; import { MobileFilterSheet, MobileFilterSheetSection } from '../shared/MobileFilterSheet'; import { ProtocolTag } from '../shared/ProtocolTag'; @@ -341,7 +341,7 @@ const AlertEventTable = memo(function AlertEventTable({ rows, selectedID, compac }, { title: '协议', dataIndex: 'protocol', className: 'is-protocol', width: 98, render: (value: string) => }, { title: '发生时间', dataIndex: 'triggeredAt', className: 'is-triggered-at', width: 124, render: (value: string) => }, - { title: '匹配自动化', dataIndex: 'ruleName', className: 'is-rule', width: 154, render: (value: string) => {value} }, + { title: '事件来源 / 自动化', dataIndex: 'ruleName', className: 'is-rule', width: 154, render: (value: string, event: AlertEvent) => {isNativeAlarm(event) ? '车辆原生告警' : value} }, { title: '执行状态', dataIndex: 'status', className: 'is-status', width: 92, render: (_: AlertEvent['status'], event: AlertEvent) => }, ]; return compact @@ -390,14 +390,22 @@ function EventInspector({ event, loading = false, detailError, note, acting, act
协议来源 发生时间 - 匹配自动化v{event.ruleVersion} + {isNativeAlarm(event) ? '事件来源' : '匹配自动化'}{isNativeAlarm(event) ? '车辆原生告警' : `v${event.ruleVersion}`}
-
+ {isNativeAlarm(event) ?
+
+ 具体告警信息 + {event.nativeAlarmNames?.length ?
    {event.nativeAlarmNames.map((name) =>
  • {name}
    • )}
:

{event.ruleName}

} + {event.nativeAlarmReservedBits?.length ?

预留位 {event.nativeAlarmReservedBits.map((bit) => `bit${bit}`).join('、')} 置位，未计入具体告警。

: null} +
+

触发时车辆上报的原始字段

+ {event.nativeAlarmFields ? :

{loading ? '正在读取告警字段…' : '告警字段暂不可用'}

} +
:
观测值{alertValue(event)} 匹配条件{thresholdText(event)} 变化幅度{alertDeltaText(event)} -
-
自动化{event.ruleName}
+
} +
{isNativeAlarm(event) ? '车辆原生告警' : '自动化'}{event.ruleName}
{normalized.execution.requiresAttention ? 处理说明{editable ? '选填' : '只读'}} headerLine aria-label="事件处置操作"> {editable ? <>