379 lines
12 KiB
JavaScript
379 lines
12 KiB
JavaScript
import fs from "node:fs/promises";
|
||
import fsSync from "node:fs";
|
||
import path from "node:path";
|
||
import readline from "node:readline";
|
||
import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool";
|
||
|
||
const sourceDir =
|
||
"/Users/lingniu/Library/Mobile Documents/com~apple~CloudDocs/rsync/2026/07/27";
|
||
const workDir =
|
||
"/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/tmp/gps-mileage-import-019fa1ac-db39-7933-bb0e-30dd63cc25bd";
|
||
const priorCsv =
|
||
"/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/g7-mileage-history-20260724/merged/G7车辆每日里程_20220101-20260720.csv";
|
||
const normalizedCsv = path.join(
|
||
workDir,
|
||
"G7_GPS车辆每日里程_20251231-20260630.csv",
|
||
);
|
||
const combinedCsv = path.join(
|
||
workDir,
|
||
"G7_GPS车辆每日里程_合并优先季度统计_20251231-20260630.csv",
|
||
);
|
||
const statisticCsv = path.join(
|
||
workDir,
|
||
"G7_GPS季度统计每日里程_20260101-20260630.csv",
|
||
);
|
||
|
||
const dailyNames = [
|
||
"智能管车_车辆里程日报-091555330.xlsx",
|
||
"智能管车_车辆里程日报-090938059.xlsx",
|
||
"智能管车_车辆里程日报-091002580.xlsx",
|
||
"智能管车_车辆里程日报-091020404.xlsx",
|
||
"智能管车_车辆里程日报-091030619.xlsx",
|
||
"智能管车_车辆里程日报-091046118.xlsx",
|
||
"智能管车_车辆里程日报-091058580.xlsx",
|
||
];
|
||
const statisticNames = [
|
||
"里程统计[天][2026-01-01至2026-03-31] (1).xlsx",
|
||
"里程统计[天][2026-01-01至2026-03-31].xlsx",
|
||
"里程统计[天][2026-04-01至2026-06-30] (1).xlsx",
|
||
"里程统计[天][2026-04-01至2026-06-30].xlsx",
|
||
];
|
||
|
||
const rows = [];
|
||
const newByKey = new Map();
|
||
const dailyFiles = [];
|
||
for (const name of dailyNames) {
|
||
const values = await readFirstSheetValues(path.join(sourceDir, name));
|
||
const headers = values[0] ?? [];
|
||
const runtimeIndex = headers.indexOf("运行时长");
|
||
assert(runtimeIndex > 3, `${name} 缺少运行时长列`);
|
||
const dateHeaders = headers.slice(3, runtimeIndex);
|
||
const dates = dateHeaders.map((header) => parseHeaderDate(header));
|
||
const seenPlates = new Set();
|
||
let statedTotalKm = 0;
|
||
let calculatedTotalKm = 0;
|
||
let mismatchRows = 0;
|
||
for (const row of values.slice(1)) {
|
||
const plate = cleanText(row?.[0]);
|
||
if (!plate) continue;
|
||
assert(!seenPlates.has(plate), `${name} 存在重复车牌 ${plate}`);
|
||
seenPlates.add(plate);
|
||
const stated = numberValue(row?.[2], 10_000_000);
|
||
const dailyValues = row
|
||
.slice(3, runtimeIndex)
|
||
.map((value) => numberValue(value));
|
||
const calculated = dailyValues.reduce((sum, value) => sum + value, 0);
|
||
statedTotalKm += stated;
|
||
calculatedTotalKm += calculated;
|
||
if (Math.abs(stated - calculated) > 0.011) mismatchRows += 1;
|
||
for (let index = 0; index < dates.length; index += 1) {
|
||
const date = dates[index];
|
||
const mileage = dailyValues[index];
|
||
const key = `${plate}|${date}`;
|
||
assert(!newByKey.has(key), `日报重复车辆日 ${key}`);
|
||
const normalized = { plate, date, mileage };
|
||
newByKey.set(key, normalized);
|
||
rows.push(normalized);
|
||
}
|
||
}
|
||
dailyFiles.push({
|
||
name,
|
||
vehicleCount: seenPlates.size,
|
||
dateFrom: dates[0],
|
||
dateTo: dates.at(-1),
|
||
dateCount: dates.length,
|
||
rowCount: seenPlates.size * dates.length,
|
||
statedTotalKm: round(statedTotalKm),
|
||
calculatedTotalKm: round(calculatedTotalKm),
|
||
mismatchRows,
|
||
});
|
||
}
|
||
|
||
rows.sort(
|
||
(a, b) =>
|
||
a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date),
|
||
);
|
||
await fs.writeFile(
|
||
normalizedCsv,
|
||
`\uFEFFplate,date,daily_mileage_km\n${rows
|
||
.map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`)
|
||
.join("\n")}\n`,
|
||
"utf8",
|
||
);
|
||
|
||
const statisticByKey = new Map();
|
||
const statisticFiles = [];
|
||
for (const name of statisticNames) {
|
||
const values = await readFirstSheetValues(path.join(sourceDir, name));
|
||
const headers = (values[1] ?? []).map(cleanText);
|
||
const plateIndex = headers.indexOf("车牌号码");
|
||
const methodIndex = headers.indexOf("里程计算方式");
|
||
const dateIndex = headers.indexOf("日期");
|
||
const mileageIndex = headers.indexOf("里程（km）");
|
||
assert(
|
||
[plateIndex, methodIndex, dateIndex, mileageIndex].every((index) => index >= 0),
|
||
`${name} 表头不完整`,
|
||
);
|
||
const fileKeys = new Set();
|
||
const methods = new Set();
|
||
let totalKm = 0;
|
||
for (const row of values.slice(2)) {
|
||
const plate = cleanText(row?.[plateIndex]);
|
||
const date = isoDate(row?.[dateIndex]);
|
||
if (!plate || !date) continue;
|
||
const method = cleanText(row?.[methodIndex]);
|
||
const mileage = numberValue(row?.[mileageIndex]);
|
||
const key = `${plate}|${date}`;
|
||
assert(!fileKeys.has(key), `${name} 文件内重复车辆日 ${key}`);
|
||
assert(!statisticByKey.has(key), `季度统计跨文件重复车辆日 ${key}`);
|
||
fileKeys.add(key);
|
||
statisticByKey.set(key, { plate, date, mileage, method, name });
|
||
methods.add(method);
|
||
totalKm += mileage;
|
||
}
|
||
statisticFiles.push({
|
||
name,
|
||
rowCount: fileKeys.size,
|
||
vehicleCount: new Set([...fileKeys].map((key) => key.split("|")[0])).size,
|
||
dateCount: new Set([...fileKeys].map((key) => key.split("|")[1])).size,
|
||
methods: [...methods].sort(),
|
||
totalKm: round(totalKm),
|
||
});
|
||
}
|
||
|
||
let statisticMatched = 0;
|
||
let statisticMissingInDaily = 0;
|
||
let statisticChanged = 0;
|
||
let statisticAbsoluteDifferenceKm = 0;
|
||
let statisticMaxDifferenceKm = 0;
|
||
const statisticExamples = [];
|
||
for (const [key, statistic] of statisticByKey) {
|
||
const daily = newByKey.get(key);
|
||
if (!daily) {
|
||
statisticMissingInDaily += 1;
|
||
continue;
|
||
}
|
||
statisticMatched += 1;
|
||
const difference = daily.mileage - statistic.mileage;
|
||
const absolute = Math.abs(difference);
|
||
statisticAbsoluteDifferenceKm += absolute;
|
||
statisticMaxDifferenceKm = Math.max(statisticMaxDifferenceKm, absolute);
|
||
if (absolute > 0.011) {
|
||
statisticChanged += 1;
|
||
if (statisticExamples.length < 20) {
|
||
statisticExamples.push({
|
||
key,
|
||
dailyKm: daily.mileage,
|
||
statisticKm: statistic.mileage,
|
||
differenceKm: round(difference),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
const statisticRows = [...statisticByKey.values()].sort(
|
||
(a, b) =>
|
||
a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date),
|
||
);
|
||
await fs.writeFile(
|
||
statisticCsv,
|
||
`\uFEFFplate,date,daily_mileage_km\n${statisticRows
|
||
.map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`)
|
||
.join("\n")}\n`,
|
||
"utf8",
|
||
);
|
||
|
||
const combinedByKey = new Map(newByKey);
|
||
for (const [key, statistic] of statisticByKey) {
|
||
combinedByKey.set(key, {
|
||
plate: statistic.plate,
|
||
date: statistic.date,
|
||
mileage: statistic.mileage,
|
||
});
|
||
}
|
||
const combinedRows = [...combinedByKey.values()].sort(
|
||
(a, b) =>
|
||
a.plate.localeCompare(b.plate, "zh-CN") || a.date.localeCompare(b.date),
|
||
);
|
||
await fs.writeFile(
|
||
combinedCsv,
|
||
`\uFEFFplate,date,daily_mileage_km\n${combinedRows
|
||
.map((row) => `${row.plate},${row.date},${formatNumber(row.mileage)}`)
|
||
.join("\n")}\n`,
|
||
"utf8",
|
||
);
|
||
|
||
const priorByKey = await loadPriorRange(
|
||
priorCsv,
|
||
"2025-12-31",
|
||
"2026-06-30",
|
||
);
|
||
let priorUnchanged = 0;
|
||
let priorChanged = 0;
|
||
let priorMissing = 0;
|
||
let priorExtra = 0;
|
||
let priorTotalDifferenceKm = 0;
|
||
let priorMaxDifferenceKm = 0;
|
||
const priorExamples = [];
|
||
for (const [key, row] of newByKey) {
|
||
const prior = priorByKey.get(key);
|
||
if (!prior) {
|
||
priorMissing += 1;
|
||
continue;
|
||
}
|
||
const difference = row.mileage - prior.mileage;
|
||
const absolute = Math.abs(difference);
|
||
priorTotalDifferenceKm += difference;
|
||
priorMaxDifferenceKm = Math.max(priorMaxDifferenceKm, absolute);
|
||
if (absolute <= 0.0005) {
|
||
priorUnchanged += 1;
|
||
} else {
|
||
priorChanged += 1;
|
||
if (priorExamples.length < 20) {
|
||
priorExamples.push({
|
||
key,
|
||
newKm: row.mileage,
|
||
priorKm: prior.mileage,
|
||
differenceKm: round(difference),
|
||
});
|
||
}
|
||
}
|
||
}
|
||
for (const key of priorByKey.keys()) {
|
||
if (!newByKey.has(key)) priorExtra += 1;
|
||
}
|
||
|
||
const result = {
|
||
normalizedCsv,
|
||
combinedCsv,
|
||
statisticCsv,
|
||
source: {
|
||
rowCount: rows.length,
|
||
vehicleCount: new Set(rows.map((row) => row.plate)).size,
|
||
dateCount: new Set(rows.map((row) => row.date)).size,
|
||
dateFrom: rows.reduce(
|
||
(min, row) => (!min || row.date < min ? row.date : min),
|
||
"",
|
||
),
|
||
dateTo: rows.reduce((max, row) => (row.date > max ? row.date : max), ""),
|
||
positiveRows: rows.filter((row) => row.mileage > 0).length,
|
||
zeroRows: rows.filter((row) => row.mileage === 0).length,
|
||
totalKm: round(rows.reduce((sum, row) => sum + row.mileage, 0)),
|
||
files: dailyFiles,
|
||
},
|
||
quarterlyReconciliation: {
|
||
rowCount: statisticByKey.size,
|
||
matchedRows: statisticMatched,
|
||
missingInDailyRows: statisticMissingInDaily,
|
||
changedRowsOver0011Km: statisticChanged,
|
||
absoluteDifferenceKm: round(statisticAbsoluteDifferenceKm),
|
||
maxDifferenceKm: round(statisticMaxDifferenceKm),
|
||
examples: statisticExamples,
|
||
files: statisticFiles,
|
||
},
|
||
combinedImport: {
|
||
rowCount: combinedRows.length,
|
||
vehicleCount: new Set(combinedRows.map((row) => row.plate)).size,
|
||
dateCount: new Set(combinedRows.map((row) => row.date)).size,
|
||
dateFrom: combinedRows.reduce(
|
||
(min, row) => (!min || row.date < min ? row.date : min),
|
||
"",
|
||
),
|
||
dateTo: combinedRows.reduce(
|
||
(max, row) => (row.date > max ? row.date : max),
|
||
"",
|
||
),
|
||
positiveRows: combinedRows.filter((row) => row.mileage > 0).length,
|
||
zeroRows: combinedRows.filter((row) => row.mileage === 0).length,
|
||
totalKm: round(
|
||
combinedRows.reduce((sum, row) => sum + row.mileage, 0),
|
||
),
|
||
precedence:
|
||
"季度里程统计（明确标注终端里程/经纬度）优先；无季度记录时使用车辆里程日报",
|
||
},
|
||
priorImportComparison: {
|
||
priorRangeRows: priorByKey.size,
|
||
unchangedRows: priorUnchanged,
|
||
changedRows: priorChanged,
|
||
missingInPriorRows: priorMissing,
|
||
extraPriorRows: priorExtra,
|
||
totalDifferenceKm: round(priorTotalDifferenceKm),
|
||
maxDifferenceKm: round(priorMaxDifferenceKm),
|
||
examples: priorExamples,
|
||
},
|
||
};
|
||
|
||
await fs.writeFile(
|
||
path.join(workDir, "normalization-manifest.json"),
|
||
`${JSON.stringify(result, null, 2)}\n`,
|
||
"utf8",
|
||
);
|
||
console.log(JSON.stringify(result, null, 2));
|
||
|
||
async function readFirstSheetValues(file) {
|
||
const input = await FileBlob.load(file);
|
||
const workbook = await SpreadsheetFile.importXlsx(input);
|
||
return workbook.worksheets.getItemAt(0).getUsedRange(true)?.values ?? [];
|
||
}
|
||
|
||
async function loadPriorRange(file, dateFrom, dateTo) {
|
||
const result = new Map();
|
||
const stream = fsSync.createReadStream(file, { encoding: "utf8" });
|
||
const lines = readline.createInterface({ input: stream, crlfDelay: Infinity });
|
||
let first = true;
|
||
for await (const line of lines) {
|
||
if (first) {
|
||
first = false;
|
||
continue;
|
||
}
|
||
if (!line) continue;
|
||
const [plate, date, rawMileage] = line.split(",");
|
||
if (date < dateFrom || date > dateTo) continue;
|
||
result.set(`${plate}|${date}`, {
|
||
plate,
|
||
date,
|
||
mileage: numberValue(rawMileage),
|
||
});
|
||
}
|
||
return result;
|
||
}
|
||
|
||
function parseHeaderDate(value) {
|
||
const text = cleanText(value);
|
||
const match = text.match(/^(\d{2})月(\d{2})日$/);
|
||
assert(match, `无法解析日报日期表头 ${text}`);
|
||
const month = Number(match[1]);
|
||
const year = month === 12 ? 2025 : 2026;
|
||
return `${year}-${match[1]}-${match[2]}`;
|
||
}
|
||
|
||
function isoDate(value) {
|
||
if (value instanceof Date) return value.toISOString().slice(0, 10);
|
||
const text = cleanText(value);
|
||
const match = text.match(/^(\d{4})-(\d{2})-(\d{2})/);
|
||
return match ? `${match[1]}-${match[2]}-${match[3]}` : "";
|
||
}
|
||
|
||
function cleanText(value) {
|
||
return String(value ?? "").trim();
|
||
}
|
||
|
||
function numberValue(value, max = 2500) {
|
||
if (value === null || value === undefined || value === "") return 0;
|
||
const number = Number(String(value).replace(/,/g, ""));
|
||
assert(Number.isFinite(number) && number >= 0 && number <= max, `非法里程 ${value}`);
|
||
return number;
|
||
}
|
||
|
||
function formatNumber(value) {
|
||
if (Number.isInteger(value)) return String(value);
|
||
return String(Number(value.toFixed(3)));
|
||
}
|
||
|
||
function round(value) {
|
||
return Math.round((value + Number.EPSILON) * 1000) / 1000;
|
||
}
|
||
|
||
function assert(condition, message) {
|
||
if (!condition) throw new Error(message);
|
||
}
|