chore: snapshot production code before Apple Design UI refinement
This commit is contained in:
lingniu
@@ -0,0 +1,741 @@
|
||||
import fs from "node:fs/promises";
|
||||
import fsSync from "node:fs";
|
||||
import path from "node:path";
|
||||
import { once } from "node:events";
|
||||
import {
|
||||
FileBlob,
|
||||
SpreadsheetFile,
|
||||
Workbook,
|
||||
} from "@oai/artifact-tool";
|
||||
|
||||
const root =
|
||||
"/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest";
|
||||
const workDir = path.join(root, "outputs/g7-mileage-history-20260724");
|
||||
const dailyDir = path.join(workDir, "raw/daily");
|
||||
const statisticDir = path.join(workDir, "raw/statistic");
|
||||
const mergedDir = path.join(workDir, "merged");
|
||||
const outputXlsx = path.join(
|
||||
mergedDir,
|
||||
"G7车辆每日里程及里程统计_20220101-20260720.xlsx",
|
||||
);
|
||||
const importCsv = path.join(
|
||||
mergedDir,
|
||||
"G7车辆每日里程_20220101-20260720.csv",
|
||||
);
|
||||
const manifestPath = path.join(mergedDir, "manifest.json");
|
||||
const priorMappingCsv = path.join(
|
||||
root,
|
||||
"outputs/g7-mileage-import-20260716/import_rows.csv",
|
||||
);
|
||||
|
||||
await fs.mkdir(mergedDir, { recursive: true });
|
||||
|
||||
const months = buildMonths();
|
||||
const dailyFilesByTaskId = await sortedXlsxFiles(dailyDir);
|
||||
// 2022-02 was the one-file validation task. The remaining formal queue then
|
||||
// started with 2022-01 and continued from 2022-03, so swap the first two task IDs
|
||||
// back into natural month order.
|
||||
const dailyFiles = [
|
||||
dailyFilesByTaskId[1],
|
||||
dailyFilesByTaskId[0],
|
||||
...dailyFilesByTaskId.slice(2),
|
||||
];
|
||||
const statisticFiles = await sortedXlsxFiles(statisticDir);
|
||||
assert(dailyFiles.length === months.length, `日报文件应为 ${months.length}，实际 ${dailyFiles.length}`);
|
||||
assert(
|
||||
statisticFiles.length === months.length,
|
||||
`里程统计文件应为 ${months.length}，实际 ${statisticFiles.length}`,
|
||||
);
|
||||
|
||||
const vehicles = new Map();
|
||||
const dateOrder = [];
|
||||
const dateSource = new Map();
|
||||
const monthlyDailyTotals = new Map();
|
||||
const dailyBatchManifest = [];
|
||||
let dailyPositiveCells = 0;
|
||||
let dailyZeroCells = 0;
|
||||
let dailyTotalKm = 0;
|
||||
let dailyTotalMismatchRows = 0;
|
||||
let dailyCompletenessNoteRows = 0;
|
||||
|
||||
for (let monthIndex = 0; monthIndex < months.length; monthIndex += 1) {
|
||||
const month = months[monthIndex];
|
||||
const file = dailyFiles[monthIndex];
|
||||
const values = await readFirstSheetValues(file);
|
||||
const headers = values[0] ?? [];
|
||||
const runtimeIndex = headers.indexOf("运行时长");
|
||||
assert(runtimeIndex > 3, `${path.basename(file)} 缺少运行时长列`);
|
||||
const dateColumnCount = runtimeIndex - 3;
|
||||
assert(
|
||||
dateColumnCount === month.dayCount,
|
||||
`${path.basename(file)} 日期列应为 ${month.dayCount}，实际 ${dateColumnCount}`,
|
||||
);
|
||||
|
||||
const monthDates = [];
|
||||
for (let day = 1; day <= month.dayCount; day += 1) {
|
||||
const date = `${month.year}-${pad(month.month)}-${pad(day)}`;
|
||||
monthDates.push(date);
|
||||
dateOrder.push(date);
|
||||
dateSource.set(date, path.basename(file));
|
||||
}
|
||||
|
||||
let batchTotal = 0;
|
||||
let batchPositive = 0;
|
||||
let batchZero = 0;
|
||||
let batchMismatch = 0;
|
||||
let batchNotes = 0;
|
||||
const seenPlates = new Set();
|
||||
for (let rowIndex = 1; rowIndex < values.length; rowIndex += 1) {
|
||||
const row = values[rowIndex] ?? [];
|
||||
const plate = cleanText(row[0]);
|
||||
if (!plate) continue;
|
||||
assert(!seenPlates.has(plate), `${path.basename(file)} 存在重复车牌 ${plate}`);
|
||||
seenPlates.add(plate);
|
||||
const organization = cleanText(row[1]);
|
||||
const statedTotal = numberValue(row[2]);
|
||||
const dailyValues = row.slice(3, runtimeIndex).map(numberValue);
|
||||
const calculatedTotal = dailyValues.reduce((sum, value) => sum + value, 0);
|
||||
if (Math.abs(statedTotal - calculatedTotal) > 0.011) {
|
||||
dailyTotalMismatchRows += 1;
|
||||
batchMismatch += 1;
|
||||
}
|
||||
const note = cleanText(row[runtimeIndex + 1]);
|
||||
if (note) {
|
||||
dailyCompletenessNoteRows += 1;
|
||||
batchNotes += 1;
|
||||
}
|
||||
|
||||
let vehicle = vehicles.get(plate);
|
||||
if (!vehicle) {
|
||||
vehicle = {
|
||||
plate,
|
||||
organization,
|
||||
values: new Map(),
|
||||
};
|
||||
vehicles.set(plate, vehicle);
|
||||
} else if (!vehicle.organization && organization) {
|
||||
vehicle.organization = organization;
|
||||
}
|
||||
|
||||
for (let dayIndex = 0; dayIndex < dailyValues.length; dayIndex += 1) {
|
||||
const value = dailyValues[dayIndex];
|
||||
const date = monthDates[dayIndex];
|
||||
vehicle.values.set(date, value);
|
||||
batchTotal += value;
|
||||
dailyTotalKm += value;
|
||||
if (value > 0) {
|
||||
batchPositive += 1;
|
||||
dailyPositiveCells += 1;
|
||||
} else {
|
||||
batchZero += 1;
|
||||
dailyZeroCells += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
monthlyDailyTotals.set(month.key, round(batchTotal));
|
||||
dailyBatchManifest.push({
|
||||
type: "车辆里程日报",
|
||||
month: month.key,
|
||||
date_from: month.dateFrom,
|
||||
date_to: month.dateTo,
|
||||
file: path.basename(file),
|
||||
vehicle_rows: seenPlates.size,
|
||||
date_columns: month.dayCount,
|
||||
positive_cells: batchPositive,
|
||||
zero_cells: batchZero,
|
||||
total_km: round(batchTotal),
|
||||
total_mismatch_rows: batchMismatch,
|
||||
completeness_note_rows: batchNotes,
|
||||
});
|
||||
}
|
||||
|
||||
assert(dateOrder.length === 1662, `日期列应为 1662，实际 ${dateOrder.length}`);
|
||||
assert(new Set(dateOrder).size === dateOrder.length, "日报日期列存在重复");
|
||||
assert(dateOrder[0] === "2022-01-01", `起始日期异常：${dateOrder[0]}`);
|
||||
assert(dateOrder.at(-1) === "2026-07-20", `结束日期异常：${dateOrder.at(-1)}`);
|
||||
|
||||
const statisticRows = [];
|
||||
const monthlyStatisticTotals = new Map();
|
||||
const statisticBatchManifest = [];
|
||||
for (let monthIndex = 0; monthIndex < months.length; monthIndex += 1) {
|
||||
const month = months[monthIndex];
|
||||
const file = statisticFiles[monthIndex];
|
||||
const values = await readFirstSheetValues(file);
|
||||
const headers = (values[0] ?? []).map(cleanText);
|
||||
const plateIndex = findHeader(headers, ["车牌号"]);
|
||||
const orgIndex = findHeader(headers, ["所属机构", "机构"]);
|
||||
const sourceOrgIndex = findHeader(headers, ["来源机构"]);
|
||||
const lengthIndex = findHeader(headers, ["车长(米)", "车长"]);
|
||||
const boxIndex = findHeader(headers, ["厢型"]);
|
||||
const runtimeIndex = findHeader(headers, ["运行时长"]);
|
||||
const mileageIndex = findHeader(headers, ["行驶里程(KM)", "行驶里程"]);
|
||||
assert(plateIndex >= 0, `${path.basename(file)} 缺少车牌号`);
|
||||
assert(mileageIndex >= 0, `${path.basename(file)} 缺少行驶里程`);
|
||||
|
||||
let batchTotal = 0;
|
||||
let vehicleRows = 0;
|
||||
for (let rowIndex = 1; rowIndex < values.length; rowIndex += 1) {
|
||||
const row = values[rowIndex] ?? [];
|
||||
const plate = cleanText(row[plateIndex]);
|
||||
if (!plate) continue;
|
||||
const mileage = numberValue(row[mileageIndex]);
|
||||
batchTotal += mileage;
|
||||
vehicleRows += 1;
|
||||
statisticRows.push([
|
||||
month.key,
|
||||
plate,
|
||||
orgIndex >= 0 ? cleanText(row[orgIndex]) : "",
|
||||
sourceOrgIndex >= 0 ? cleanText(row[sourceOrgIndex]) : "",
|
||||
lengthIndex >= 0 ? row[lengthIndex] ?? "" : "",
|
||||
boxIndex >= 0 ? cleanText(row[boxIndex]) : "",
|
||||
runtimeIndex >= 0 ? cleanText(row[runtimeIndex]) : "",
|
||||
mileage,
|
||||
path.basename(file),
|
||||
]);
|
||||
}
|
||||
monthlyStatisticTotals.set(month.key, round(batchTotal));
|
||||
statisticBatchManifest.push({
|
||||
type: "车辆里程统计",
|
||||
month: month.key,
|
||||
date_from: month.dateFrom,
|
||||
date_to: month.dateTo,
|
||||
file: path.basename(file),
|
||||
vehicle_rows: vehicleRows,
|
||||
total_km: round(batchTotal),
|
||||
});
|
||||
}
|
||||
|
||||
const mapping = await readPriorMapping(priorMappingCsv);
|
||||
const sortedVehicles = [...vehicles.values()].sort((a, b) =>
|
||||
a.plate.localeCompare(b.plate, "zh-CN"),
|
||||
);
|
||||
const mappedVehicles = sortedVehicles.filter((vehicle) => mapping.has(vehicle.plate));
|
||||
const unmappedVehicles = sortedVehicles.filter((vehicle) => !mapping.has(vehicle.plate));
|
||||
|
||||
await writeImportCsv(importCsv, sortedVehicles, dateOrder);
|
||||
|
||||
const workbook = Workbook.create();
|
||||
const summarySheet = workbook.worksheets.add("汇总");
|
||||
const positiveDailySheet = workbook.worksheets.add("每日里程非零");
|
||||
const statisticSheet = workbook.worksheets.add("G7里程统计");
|
||||
const mappingSheet = workbook.worksheets.add("车辆映射");
|
||||
const batchSheet = workbook.worksheets.add("导出批次");
|
||||
|
||||
buildSummarySheet(summarySheet);
|
||||
await buildPositiveDailySheet(positiveDailySheet);
|
||||
await buildStatisticSheet(statisticSheet);
|
||||
await buildMappingSheet(mappingSheet);
|
||||
await buildBatchSheet(batchSheet);
|
||||
|
||||
const summaryInspect = await workbook.inspect({
|
||||
kind: "table",
|
||||
sheetId: "汇总",
|
||||
range: "A1:F25",
|
||||
include: "values,formulas",
|
||||
maxChars: 12000,
|
||||
tableMaxRows: 25,
|
||||
tableMaxCols: 6,
|
||||
});
|
||||
const dailyInspect = await workbook.inspect({
|
||||
kind: "table",
|
||||
sheetId: "每日里程非零",
|
||||
range: "A1:D12",
|
||||
include: "values,formulas",
|
||||
maxChars: 12000,
|
||||
tableMaxRows: 12,
|
||||
tableMaxCols: 4,
|
||||
});
|
||||
const formulaErrors = await workbook.inspect({
|
||||
kind: "match",
|
||||
searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A",
|
||||
options: { useRegex: true, maxResults: 300 },
|
||||
summary: "final formula error scan",
|
||||
});
|
||||
assert(!formulaErrors.ndjson.includes('"match"'), "工作簿存在公式错误");
|
||||
|
||||
const summaryPreview = await workbook.render({
|
||||
sheetName: "汇总",
|
||||
range: "A1:F25",
|
||||
scale: 1.5,
|
||||
format: "png",
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(mergedDir, "summary-preview.png"),
|
||||
new Uint8Array(await summaryPreview.arrayBuffer()),
|
||||
);
|
||||
const dailyPreview = await workbook.render({
|
||||
sheetName: "每日里程非零",
|
||||
range: "A1:D12",
|
||||
scale: 1.2,
|
||||
format: "png",
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(mergedDir, "daily-preview.png"),
|
||||
new Uint8Array(await dailyPreview.arrayBuffer()),
|
||||
);
|
||||
|
||||
try {
|
||||
const output = await SpreadsheetFile.exportXlsx(workbook);
|
||||
await output.save(outputXlsx);
|
||||
} catch (error) {
|
||||
console.error(
|
||||
JSON.stringify(
|
||||
{
|
||||
stage: "export_xlsx",
|
||||
name: error?.name,
|
||||
message: error?.message,
|
||||
stack: String(error?.stack ?? "")
|
||||
.split("\n")
|
||||
.filter((line) => !line.includes("artifact_tool.mjs:3121"))
|
||||
.slice(0, 12),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const monthlyReconciliation = months.map((month) => {
|
||||
const dailyKm = monthlyDailyTotals.get(month.key) ?? 0;
|
||||
const statisticKm = monthlyStatisticTotals.get(month.key) ?? 0;
|
||||
return {
|
||||
month: month.key,
|
||||
daily_km: dailyKm,
|
||||
statistic_km: statisticKm,
|
||||
difference_km: round(dailyKm - statisticKm),
|
||||
};
|
||||
});
|
||||
const manifest = {
|
||||
generated_at: new Date().toISOString(),
|
||||
range: {
|
||||
date_from: dateOrder[0],
|
||||
date_to: dateOrder.at(-1),
|
||||
days: dateOrder.length,
|
||||
months: months.length,
|
||||
},
|
||||
files: {
|
||||
daily_raw: dailyFiles.map((file) => path.basename(file)),
|
||||
statistic_raw: statisticFiles.map((file) => path.basename(file)),
|
||||
workbook: path.basename(outputXlsx),
|
||||
import_csv: path.basename(importCsv),
|
||||
},
|
||||
vehicle_count: sortedVehicles.length,
|
||||
mapped_vehicle_count: mappedVehicles.length,
|
||||
unmapped_vehicle_count: unmappedVehicles.length,
|
||||
vehicle_day_rows: sortedVehicles.length * dateOrder.length,
|
||||
mapped_vehicle_day_rows: mappedVehicles.length * dateOrder.length,
|
||||
positive_cells: dailyPositiveCells,
|
||||
zero_cells: dailyZeroCells,
|
||||
daily_total_km: round(dailyTotalKm),
|
||||
daily_total_mismatch_rows: dailyTotalMismatchRows,
|
||||
completeness_note_rows: dailyCompletenessNoteRows,
|
||||
monthly_reconciliation: monthlyReconciliation,
|
||||
daily_batches: dailyBatchManifest,
|
||||
statistic_batches: statisticBatchManifest,
|
||||
verification: {
|
||||
summary_inspect: summaryInspect.ndjson,
|
||||
daily_inspect: dailyInspect.ndjson,
|
||||
formula_errors: formulaErrors.ndjson,
|
||||
},
|
||||
};
|
||||
await fs.writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
outputXlsx,
|
||||
importCsv,
|
||||
manifestPath,
|
||||
vehicleCount: sortedVehicles.length,
|
||||
mappedVehicleCount: mappedVehicles.length,
|
||||
unmappedVehicleCount: unmappedVehicles.length,
|
||||
dateCount: dateOrder.length,
|
||||
vehicleDayRows: sortedVehicles.length * dateOrder.length,
|
||||
mappedVehicleDayRows: mappedVehicles.length * dateOrder.length,
|
||||
positiveCells: dailyPositiveCells,
|
||||
zeroCells: dailyZeroCells,
|
||||
totalKm: round(dailyTotalKm),
|
||||
monthlyDifferenceCount: monthlyReconciliation.filter(
|
||||
(row) => Math.abs(row.difference_km) > 0.02,
|
||||
).length,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
function buildSummarySheet(sheet) {
|
||||
sheet.showGridLines = false;
|
||||
sheet.getRange("A1:F1").merge();
|
||||
sheet.getRange("A1").values = [["G7 车辆里程全量导出汇总"]];
|
||||
sheet.getRange("A1:F1").format = {
|
||||
fill: "#0F766E",
|
||||
font: { bold: true, color: "#FFFFFF", size: 16 },
|
||||
horizontalAlignment: "center",
|
||||
verticalAlignment: "center",
|
||||
};
|
||||
sheet.getRange("A1:F1").format.rowHeight = 30;
|
||||
|
||||
const metrics = [
|
||||
["指标", "值", "说明"],
|
||||
["统计起始日期", new Date(`${dateOrder[0]}T00:00:00+08:00`), "G7 日报"],
|
||||
["统计结束日期", new Date(`${dateOrder.at(-1)}T00:00:00+08:00`), "含当日"],
|
||||
["自然日数", dateOrder.length, "连续日期"],
|
||||
["日报导出批次", dailyFiles.length, "每月 1 份"],
|
||||
["里程统计导出批次", statisticFiles.length, "每月 1 份"],
|
||||
["G7 车辆数", sortedVehicles.length, "当前导出车辆列表"],
|
||||
["已映射 VIN 车辆数", mappedVehicles.length, "沿用现有车辆映射"],
|
||||
["未映射车辆数", unmappedVehicles.length, "不进入正式导入"],
|
||||
["映射覆盖率", null, "已映射 / G7 车辆数"],
|
||||
["车辆-日期单元格", sortedVehicles.length * dateOrder.length, "包含 0 km"],
|
||||
["可导入车辆-日期行", mappedVehicles.length * dateOrder.length, "包含 0 km"],
|
||||
["正里程单元格", dailyPositiveCells, "> 0 km"],
|
||||
["0 km 单元格", dailyZeroCells, "按用户确认保留"],
|
||||
["日报累计里程(km)", round(dailyTotalKm), "55 份日报求和"],
|
||||
["日报合计不一致车辆月", dailyTotalMismatchRows, "日报“行驶里程”与逐日求和"],
|
||||
["完整度说明非空车辆月", dailyCompletenessNoteRows, "来自 G7 日报"],
|
||||
];
|
||||
sheet.getRangeByIndexes(2, 0, metrics.length, 3).values = metrics;
|
||||
sheet.getRange("A3:C3").format = headerFormat();
|
||||
sheet.getRange("B4:B5").format.numberFormat = "yyyy-mm-dd";
|
||||
sheet.getRange("B6:B19").format.numberFormat = "#,##0.00";
|
||||
sheet.getRange("B12").formulas = [["=IFERROR(B10/B9,0)"]];
|
||||
sheet.getRange("B12").format.numberFormat = "0.0%";
|
||||
sheet.getRange("A4:A19").format.font = { bold: true, color: "#134E4A" };
|
||||
sheet.getRange("A3:C19").format.borders = {
|
||||
preset: "inside",
|
||||
style: "thin",
|
||||
color: "#D1D5DB",
|
||||
};
|
||||
|
||||
sheet.getRange("E3:H3").values = [["月份", "日报合计(km)", "里程统计合计(km)", "差异(km)"]];
|
||||
sheet.getRange("E3:H3").format = headerFormat();
|
||||
const monthlyRows = months.map((month) => [
|
||||
month.key,
|
||||
monthlyDailyTotals.get(month.key) ?? 0,
|
||||
monthlyStatisticTotals.get(month.key) ?? 0,
|
||||
null,
|
||||
]);
|
||||
sheet.getRangeByIndexes(3, 4, monthlyRows.length, 4).values = monthlyRows;
|
||||
sheet.getRange("H4").formulas = [["=F4-G4"]];
|
||||
sheet.getRange(`H4:H${3 + monthlyRows.length}`).fillDown();
|
||||
sheet.getRange(`F4:H${3 + monthlyRows.length}`).format.numberFormat = "#,##0.00";
|
||||
sheet.getRange(`E3:H${3 + monthlyRows.length}`).format.borders = {
|
||||
preset: "inside",
|
||||
style: "thin",
|
||||
color: "#E5E7EB",
|
||||
};
|
||||
|
||||
sheet.freezePanes.freezeRows(3);
|
||||
sheet.getRange("A21:C23").merge();
|
||||
sheet.getRange("A21").values = [[
|
||||
"Excel 的“每日里程非零”工作表列出全部正里程记录；未出现的车辆×日期组合按用户确认均为 0 km。含全部 0 km 的逐行明细保存在同目录 CSV：G7车辆每日里程_20220101-20260720.csv。",
|
||||
]];
|
||||
sheet.getRange("A21:C23").format = {
|
||||
fill: "#ECFDF5",
|
||||
font: { color: "#065F46", italic: true },
|
||||
wrapText: true,
|
||||
verticalAlignment: "center",
|
||||
};
|
||||
sheet.getRange(`A1:H${3 + monthlyRows.length}`).format.font = { name: "Aptos", size: 10 };
|
||||
sheet.getRange(`A1:A${3 + monthlyRows.length}`).format.columnWidth = 24;
|
||||
sheet.getRange(`B1:B${3 + monthlyRows.length}`).format.columnWidth = 18;
|
||||
sheet.getRange(`C1:C${3 + monthlyRows.length}`).format.columnWidth = 30;
|
||||
sheet.getRange(`D1:D${3 + monthlyRows.length}`).format.columnWidth = 3;
|
||||
sheet.getRange(`E1:E${3 + monthlyRows.length}`).format.columnWidth = 12;
|
||||
sheet.getRange(`F1:H${3 + monthlyRows.length}`).format.columnWidth = 18;
|
||||
}
|
||||
|
||||
async function buildPositiveDailySheet(sheet) {
|
||||
sheet.showGridLines = false;
|
||||
const headers = ["日期", "车牌号", "机构", "每日里程(km)"];
|
||||
sheet.getRange("A1:D1").values = [headers];
|
||||
sheet.getRange("A1:D1").format = headerFormat();
|
||||
const rows = [];
|
||||
for (const vehicle of sortedVehicles) {
|
||||
for (const date of dateOrder) {
|
||||
const mileage = vehicle.values.get(date) ?? 0;
|
||||
if (mileage <= 0) continue;
|
||||
rows.push([
|
||||
new Date(`${date}T00:00:00+08:00`),
|
||||
vehicle.plate,
|
||||
vehicle.organization,
|
||||
mileage,
|
||||
]);
|
||||
}
|
||||
}
|
||||
const chunkSize = 1000;
|
||||
for (let start = 0; start < rows.length; start += chunkSize) {
|
||||
const chunk = rows.slice(start, start + chunkSize);
|
||||
sheet.getRangeByIndexes(start + 1, 0, chunk.length, headers.length).values = chunk;
|
||||
}
|
||||
sheet.getRange(`A2:A${rows.length + 1}`).format.numberFormat = "yyyy-mm-dd";
|
||||
sheet.getRange(`D2:D${rows.length + 1}`).format.numberFormat = "#,##0.00";
|
||||
sheet.freezePanes.freezeRows(1);
|
||||
[14, 16, 30, 18].forEach((width, index) => {
|
||||
sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width;
|
||||
});
|
||||
}
|
||||
|
||||
async function buildStatisticSheet(sheet) {
|
||||
sheet.showGridLines = false;
|
||||
const headers = [
|
||||
"月份",
|
||||
"车牌号",
|
||||
"所属机构",
|
||||
"来源机构",
|
||||
"车长(米)",
|
||||
"厢型",
|
||||
"运行时长",
|
||||
"行驶里程(KM)",
|
||||
"来源文件",
|
||||
];
|
||||
sheet.getRange("A1:I1").values = [headers];
|
||||
sheet.getRange("A1:I1").format = headerFormat();
|
||||
const chunkSize = 1000;
|
||||
for (let start = 0; start < statisticRows.length; start += chunkSize) {
|
||||
const chunk = statisticRows.slice(start, start + chunkSize);
|
||||
sheet.getRangeByIndexes(start + 1, 0, chunk.length, headers.length).values = chunk;
|
||||
}
|
||||
sheet.getRange(`E2:E${statisticRows.length + 1}`).format.numberFormat = "0.0";
|
||||
sheet.getRange(`H2:H${statisticRows.length + 1}`).format.numberFormat = "#,##0.00";
|
||||
sheet.freezePanes.freezeRows(1);
|
||||
const widths = [12, 16, 28, 28, 12, 16, 18, 18, 34];
|
||||
widths.forEach((width, index) => {
|
||||
sheet.getRangeByIndexes(0, index, statisticRows.length + 1, 1).format.columnWidth = width;
|
||||
});
|
||||
}
|
||||
|
||||
async function buildMappingSheet(sheet) {
|
||||
sheet.showGridLines = false;
|
||||
const headers = ["车牌号", "机构", "VIN", "终端号", "映射状态", "期间里程(km)", "正里程天数", "0 km天数"];
|
||||
sheet.getRange("A1:H1").values = [headers];
|
||||
sheet.getRange("A1:H1").format = headerFormat();
|
||||
const rows = sortedVehicles.map((vehicle) => {
|
||||
const mapped = mapping.get(vehicle.plate);
|
||||
const allValues = dateOrder.map((date) => vehicle.values.get(date) ?? 0);
|
||||
const positiveDays = allValues.filter((value) => value > 0).length;
|
||||
return [
|
||||
vehicle.plate,
|
||||
vehicle.organization,
|
||||
mapped?.vin ?? "",
|
||||
mapped?.phone ?? "",
|
||||
mapped ? "已映射" : "未映射",
|
||||
round(allValues.reduce((sum, value) => sum + value, 0)),
|
||||
positiveDays,
|
||||
dateOrder.length - positiveDays,
|
||||
];
|
||||
});
|
||||
sheet.getRangeByIndexes(1, 0, rows.length, headers.length).values = rows;
|
||||
sheet.getRange(`F2:F${rows.length + 1}`).format.numberFormat = "#,##0.00";
|
||||
sheet.getRange(`E2:E${rows.length + 1}`).conditionalFormats.add("containsText", {
|
||||
text: "未映射",
|
||||
format: { fill: "#FEE2E2", font: { color: "#991B1B", bold: true } },
|
||||
});
|
||||
sheet.freezePanes.freezeRows(1);
|
||||
[16, 30, 22, 18, 14, 18, 14, 14].forEach((width, index) => {
|
||||
sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width;
|
||||
});
|
||||
}
|
||||
|
||||
async function buildBatchSheet(sheet) {
|
||||
sheet.showGridLines = false;
|
||||
const headers = [
|
||||
"类型",
|
||||
"月份",
|
||||
"开始日期",
|
||||
"结束日期",
|
||||
"来源文件",
|
||||
"车辆数",
|
||||
"日期列数",
|
||||
"正里程单元格",
|
||||
"0 km 单元格",
|
||||
"合计里程(km)",
|
||||
];
|
||||
sheet.getRange("A1:J1").values = [headers];
|
||||
sheet.getRange("A1:J1").format = headerFormat();
|
||||
const rows = [
|
||||
...dailyBatchManifest.map((batch) => [
|
||||
batch.type,
|
||||
batch.month,
|
||||
new Date(`${batch.date_from}T00:00:00+08:00`),
|
||||
new Date(`${batch.date_to}T00:00:00+08:00`),
|
||||
batch.file,
|
||||
batch.vehicle_rows,
|
||||
batch.date_columns,
|
||||
batch.positive_cells,
|
||||
batch.zero_cells,
|
||||
batch.total_km,
|
||||
]),
|
||||
...statisticBatchManifest.map((batch) => [
|
||||
batch.type,
|
||||
batch.month,
|
||||
new Date(`${batch.date_from}T00:00:00+08:00`),
|
||||
new Date(`${batch.date_to}T00:00:00+08:00`),
|
||||
batch.file,
|
||||
batch.vehicle_rows,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
batch.total_km,
|
||||
]),
|
||||
];
|
||||
sheet.getRangeByIndexes(1, 0, rows.length, headers.length).values = rows;
|
||||
sheet.getRange(`C2:D${rows.length + 1}`).format.numberFormat = "yyyy-mm-dd";
|
||||
sheet.getRange(`F2:J${rows.length + 1}`).format.numberFormat = "#,##0.00";
|
||||
sheet.freezePanes.freezeRows(1);
|
||||
[18, 12, 14, 14, 36, 12, 12, 18, 18, 18].forEach((width, index) => {
|
||||
sheet.getRangeByIndexes(0, index, rows.length + 1, 1).format.columnWidth = width;
|
||||
});
|
||||
}
|
||||
|
||||
async function readFirstSheetValues(file) {
|
||||
const input = await FileBlob.load(file);
|
||||
const workbook = await SpreadsheetFile.importXlsx(input);
|
||||
const sheet = workbook.worksheets.getItemAt(0);
|
||||
const usedRange = sheet.getUsedRange(true);
|
||||
return usedRange?.values ?? [];
|
||||
}
|
||||
|
||||
async function sortedXlsxFiles(directory) {
|
||||
const names = (await fs.readdir(directory))
|
||||
.filter((name) => name.endsWith(".xlsx"))
|
||||
.sort((a, b) => extractTaskId(a) - extractTaskId(b));
|
||||
return names.map((name) => path.join(directory, name));
|
||||
}
|
||||
|
||||
function extractTaskId(file) {
|
||||
const match = path.basename(file).match(/-(\d+)\.xlsx$/);
|
||||
assert(match, `无法从文件名解析任务号：${file}`);
|
||||
return Number(match[1]);
|
||||
}
|
||||
|
||||
function buildMonths() {
|
||||
const result = [];
|
||||
for (let year = 2022, month = 1; year < 2026 || (year === 2026 && month <= 7); ) {
|
||||
const naturalLastDay = new Date(Date.UTC(year, month, 0)).getUTCDate();
|
||||
const lastDay = year === 2026 && month === 7 ? 20 : naturalLastDay;
|
||||
result.push({
|
||||
key: `${year}-${pad(month)}`,
|
||||
year,
|
||||
month,
|
||||
dayCount: lastDay,
|
||||
dateFrom: `${year}-${pad(month)}-01`,
|
||||
dateTo: `${year}-${pad(month)}-${pad(lastDay)}`,
|
||||
});
|
||||
month += 1;
|
||||
if (month === 13) {
|
||||
year += 1;
|
||||
month = 1;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readPriorMapping(csvPath) {
|
||||
const text = await fs.readFile(csvPath, "utf8");
|
||||
const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/);
|
||||
const result = new Map();
|
||||
for (const line of lines.slice(1)) {
|
||||
if (!line) continue;
|
||||
const columns = parseCsvLine(line);
|
||||
const [vin, plate, phone] = columns;
|
||||
if (!vin || !plate) continue;
|
||||
const current = result.get(plate);
|
||||
if (current && current.vin !== vin) {
|
||||
result.delete(plate);
|
||||
continue;
|
||||
}
|
||||
result.set(plate, { vin, phone: phone ?? "" });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function writeImportCsv(file, vehicleRows, dates) {
|
||||
const stream = fsSync.createWriteStream(file, { encoding: "utf8" });
|
||||
stream.write("\uFEFFplate,date,daily_mileage_km\n");
|
||||
for (const vehicle of vehicleRows) {
|
||||
for (const date of dates) {
|
||||
const value = vehicle.values.get(date) ?? 0;
|
||||
if (!stream.write(`${csvCell(vehicle.plate)},${date},${numberText(value)}\n`)) {
|
||||
await once(stream, "drain");
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.end();
|
||||
await once(stream, "finish");
|
||||
}
|
||||
|
||||
function headerFormat() {
|
||||
return {
|
||||
fill: "#0F766E",
|
||||
font: { bold: true, color: "#FFFFFF" },
|
||||
horizontalAlignment: "center",
|
||||
verticalAlignment: "center",
|
||||
wrapText: true,
|
||||
borders: { preset: "outside", style: "thin", color: "#115E59" },
|
||||
};
|
||||
}
|
||||
|
||||
function findHeader(headers, names) {
|
||||
return headers.findIndex((header) => names.includes(header));
|
||||
}
|
||||
|
||||
function parseCsvLine(line) {
|
||||
const result = [];
|
||||
let value = "";
|
||||
let quoted = false;
|
||||
for (let index = 0; index < line.length; index += 1) {
|
||||
const char = line[index];
|
||||
if (quoted) {
|
||||
if (char === '"' && line[index + 1] === '"') {
|
||||
value += '"';
|
||||
index += 1;
|
||||
} else if (char === '"') {
|
||||
quoted = false;
|
||||
} else {
|
||||
value += char;
|
||||
}
|
||||
} else if (char === '"') {
|
||||
quoted = true;
|
||||
} else if (char === ",") {
|
||||
result.push(value);
|
||||
value = "";
|
||||
} else {
|
||||
value += char;
|
||||
}
|
||||
}
|
||||
result.push(value);
|
||||
return result;
|
||||
}
|
||||
|
||||
function csvCell(value) {
|
||||
const text = cleanText(value);
|
||||
return /[",\r\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text;
|
||||
}
|
||||
|
||||
function cleanText(value) {
|
||||
return value === null || value === undefined ? "" : String(value).trim();
|
||||
}
|
||||
|
||||
function numberValue(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? number : 0;
|
||||
}
|
||||
|
||||
function numberText(value) {
|
||||
return String(round(value));
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round((value + Number.EPSILON) * 100) / 100;
|
||||
}
|
||||
|
||||
function pad(value) {
|
||||
return String(value).padStart(2, "0");
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) throw new Error(message);
|
||||
}
|
||||
@@ -0,0 +1,787 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
mysql "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
const (
|
||||
g7Protocol = "JT808"
|
||||
g7SourceID = int64(661200)
|
||||
g7SourceIP = "manual-g7s-excel"
|
||||
g7SourceCode = "g7s_excel"
|
||||
g7PlatformName = "G7s Excel Override"
|
||||
g7QualityReason = "manual_g7s_gps_quarterly_import_20260101_20260630_20260728"
|
||||
dateFrom = "2026-01-01"
|
||||
dateTo = "2026-06-30"
|
||||
expectedDays = 181
|
||||
)
|
||||
|
||||
type mapping struct {
|
||||
VIN string
|
||||
Ambiguous bool
|
||||
}
|
||||
|
||||
type sourceStats struct {
|
||||
Rows int64 `json:"rows"`
|
||||
Vehicles int `json:"vehicles"`
|
||||
Dates int `json:"dates"`
|
||||
Positive int64 `json:"positive_rows"`
|
||||
Zero int64 `json:"zero_rows"`
|
||||
TotalKM float64 `json:"total_km"`
|
||||
MinDate string `json:"min_date"`
|
||||
MaxDate string `json:"max_date"`
|
||||
PlateCount int `json:"plate_count"`
|
||||
}
|
||||
|
||||
type stageStats struct {
|
||||
Rows int64 `json:"rows"`
|
||||
Vehicles int `json:"vehicles"`
|
||||
Dates int `json:"dates"`
|
||||
Positive int64 `json:"positive_rows"`
|
||||
Zero int64 `json:"zero_rows"`
|
||||
TotalKM float64 `json:"total_km"`
|
||||
MappedPlates int `json:"mapped_plates"`
|
||||
UnmappedPlates int `json:"unmapped_plates"`
|
||||
AmbiguousPlates int `json:"ambiguous_plates"`
|
||||
VINCollisionPlates int `json:"vin_collision_plates"`
|
||||
}
|
||||
|
||||
type existingStats struct {
|
||||
ExistingFinalRows int64 `json:"existing_final_rows"`
|
||||
NewFinalRows int64 `json:"new_final_rows"`
|
||||
ChangedExistingFinalRows int64 `json:"changed_existing_final_rows"`
|
||||
UnchangedExistingFinalRows int64 `json:"unchanged_existing_final_rows"`
|
||||
OtherSourceFinalRows int64 `json:"other_source_final_rows"`
|
||||
OtherSelectedCandidateRows int64 `json:"other_selected_candidate_rows"`
|
||||
ExistingFinalTotalKM float64 `json:"existing_final_total_km"`
|
||||
}
|
||||
|
||||
type result struct {
|
||||
Mode string `json:"mode"`
|
||||
StartedAt string `json:"started_at"`
|
||||
CompletedAt string `json:"completed_at"`
|
||||
Source sourceStats `json:"source"`
|
||||
Stage stageStats `json:"stage"`
|
||||
Existing existingStats `json:"existing"`
|
||||
UnmappedPlates []string `json:"unmapped_plates"`
|
||||
AmbiguousPlates []string `json:"ambiguous_plates"`
|
||||
CollisionPlates []string `json:"vin_collision_plates"`
|
||||
BackupFinal string `json:"backup_final,omitempty"`
|
||||
BackupCandidates string `json:"backup_candidates,omitempty"`
|
||||
FinalMatchedRows int64 `json:"final_matched_rows"`
|
||||
FinalTotalKM float64 `json:"final_total_km"`
|
||||
CandidateMatchedRows int64 `json:"candidate_matched_rows"`
|
||||
OtherSelectedRows int64 `json:"other_selected_rows"`
|
||||
SourceID int64 `json:"source_id"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
var inputPath, mappingPath, dsn, backupDir string
|
||||
var apply bool
|
||||
var timeout time.Duration
|
||||
flag.StringVar(&inputPath, "input", "", "normalized CSV or CSV.gz")
|
||||
flag.StringVar(&mappingPath, "mapping-csv", "", "optional existing vin,plate mapping CSV")
|
||||
flag.StringVar(&dsn, "mysql-dsn", strings.TrimSpace(os.Getenv("MYSQL_DSN")), "MySQL DSN")
|
||||
flag.StringVar(&backupDir, "backup-dir", "/tmp/g7-gps-quarterly-backup-20260728", "backup directory for apply")
|
||||
flag.BoolVar(&apply, "apply", false, "apply changes; default is dry-run")
|
||||
flag.DurationVar(&timeout, "timeout", 30*time.Minute, "overall timeout")
|
||||
flag.Parse()
|
||||
|
||||
if strings.TrimSpace(inputPath) == "" {
|
||||
return errors.New("-input is required")
|
||||
}
|
||||
if strings.TrimSpace(dsn) == "" {
|
||||
return errors.New("MYSQL_DSN or -mysql-dsn is required")
|
||||
}
|
||||
normalized, err := normalizedDSN(dsn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
db, err := sql.Open("mysql", normalized)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(2)
|
||||
db.SetMaxIdleConns(2)
|
||||
if err := db.PingContext(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
started := time.Now()
|
||||
source, inputPlates, err := scanInput(inputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if source.MinDate != dateFrom || source.MaxDate != dateTo || source.Dates != expectedDays {
|
||||
return fmt.Errorf("unexpected input range: %s..%s (%d dates)", source.MinDate, source.MaxDate, source.Dates)
|
||||
}
|
||||
mappings, err := loadMappings(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if mappingPath != "" {
|
||||
if err := mergeMappingCSV(mappingPath, mappings); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
resolution := resolveInputPlates(inputPlates, mappings)
|
||||
if err := createStage(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
stage, err := loadStage(ctx, conn, inputPath, resolution)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stage.MappedPlates = len(resolution.Mapped)
|
||||
stage.UnmappedPlates = len(resolution.Unmapped)
|
||||
stage.AmbiguousPlates = len(resolution.Ambiguous)
|
||||
stage.VINCollisionPlates = len(resolution.Collisions)
|
||||
if stage.Rows != int64(stage.MappedPlates*expectedDays) {
|
||||
return fmt.Errorf("stage row count mismatch: got %d, expected %d", stage.Rows, stage.MappedPlates*expectedDays)
|
||||
}
|
||||
existing, err := inspectExisting(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := result{
|
||||
Mode: "dry_run",
|
||||
StartedAt: started.Format(time.RFC3339),
|
||||
Source: source,
|
||||
Stage: stage,
|
||||
Existing: existing,
|
||||
UnmappedPlates: resolution.Unmapped,
|
||||
AmbiguousPlates: resolution.Ambiguous,
|
||||
CollisionPlates: resolution.Collisions,
|
||||
SourceID: g7SourceID,
|
||||
}
|
||||
|
||||
if apply {
|
||||
out.Mode = "apply"
|
||||
if err := os.MkdirAll(backupDir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
out.BackupFinal = filepath.Join(backupDir, "vehicle_daily_mileage_before.csv.gz")
|
||||
out.BackupCandidates = filepath.Join(backupDir, "vehicle_daily_mileage_source_before.csv.gz")
|
||||
if err := backupQuery(ctx, conn, out.BackupFinal, `
|
||||
SELECT m.*
|
||||
FROM vehicle_daily_mileage m
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = m.vin AND s.stat_date = m.stat_date
|
||||
WHERE m.protocol = 'JT808'
|
||||
ORDER BY m.vin, m.stat_date`); err != nil {
|
||||
return fmt.Errorf("backup final rows: %w", err)
|
||||
}
|
||||
if err := backupQuery(ctx, conn, out.BackupCandidates, `
|
||||
SELECT c.*
|
||||
FROM vehicle_daily_mileage_source c
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = c.vin AND s.stat_date = c.stat_date
|
||||
WHERE c.protocol = 'JT808'
|
||||
ORDER BY c.vin, c.stat_date, c.source_key`); err != nil {
|
||||
return fmt.Errorf("backup candidate rows: %w", err)
|
||||
}
|
||||
finalRows, finalTotal, candidateRows, otherSelected, err := applyImport(ctx, conn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out.FinalMatchedRows = finalRows
|
||||
out.FinalTotalKM = finalTotal
|
||||
out.CandidateMatchedRows = candidateRows
|
||||
out.OtherSelectedRows = otherSelected
|
||||
}
|
||||
out.CompletedAt = time.Now().Format(time.RFC3339)
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(out)
|
||||
}
|
||||
|
||||
type plateResolution struct {
|
||||
Mapped map[string]string
|
||||
Unmapped []string
|
||||
Ambiguous []string
|
||||
Collisions []string
|
||||
}
|
||||
|
||||
func resolveInputPlates(plates map[string]struct{}, mappings map[string]mapping) plateResolution {
|
||||
result := plateResolution{Mapped: make(map[string]string)}
|
||||
vinToPlates := make(map[string][]string)
|
||||
for plate := range plates {
|
||||
value, ok := mappings[plate]
|
||||
switch {
|
||||
case !ok:
|
||||
result.Unmapped = append(result.Unmapped, plate)
|
||||
case value.Ambiguous:
|
||||
result.Ambiguous = append(result.Ambiguous, plate)
|
||||
default:
|
||||
result.Mapped[plate] = value.VIN
|
||||
vinToPlates[value.VIN] = append(vinToPlates[value.VIN], plate)
|
||||
}
|
||||
}
|
||||
for _, presentPlates := range vinToPlates {
|
||||
if len(presentPlates) < 2 {
|
||||
continue
|
||||
}
|
||||
for _, plate := range presentPlates {
|
||||
delete(result.Mapped, plate)
|
||||
result.Collisions = append(result.Collisions, plate)
|
||||
}
|
||||
}
|
||||
sort.Strings(result.Unmapped)
|
||||
sort.Strings(result.Ambiguous)
|
||||
sort.Strings(result.Collisions)
|
||||
return result
|
||||
}
|
||||
|
||||
func scanInput(inputPath string) (sourceStats, map[string]struct{}, error) {
|
||||
reader, closeReader, err := openCSV(inputPath)
|
||||
if err != nil {
|
||||
return sourceStats{}, nil, err
|
||||
}
|
||||
defer closeReader()
|
||||
csvReader := csv.NewReader(reader)
|
||||
header, err := csvReader.Read()
|
||||
if err != nil {
|
||||
return sourceStats{}, nil, err
|
||||
}
|
||||
if len(header) < 3 {
|
||||
return sourceStats{}, nil, errors.New("input CSV needs plate,date,daily_mileage_km")
|
||||
}
|
||||
header[0] = strings.TrimPrefix(header[0], "\uFEFF")
|
||||
if header[0] != "plate" || header[1] != "date" || header[2] != "daily_mileage_km" {
|
||||
return sourceStats{}, nil, fmt.Errorf("unexpected CSV header: %v", header)
|
||||
}
|
||||
plates := make(map[string]struct{})
|
||||
dates := make(map[string]struct{})
|
||||
stats := sourceStats{}
|
||||
for {
|
||||
record, readErr := csvReader.Read()
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return sourceStats{}, nil, readErr
|
||||
}
|
||||
if len(record) < 3 {
|
||||
return sourceStats{}, nil, fmt.Errorf("short CSV row at %d", stats.Rows+2)
|
||||
}
|
||||
plate := strings.TrimSpace(record[0])
|
||||
date := strings.TrimSpace(record[1])
|
||||
mileage, parseErr := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
|
||||
if parseErr != nil || mileage < 0 || mileage > 2500 {
|
||||
return sourceStats{}, nil, fmt.Errorf("invalid mileage at row %d: %q", stats.Rows+2, record[2])
|
||||
}
|
||||
if _, parseErr = time.Parse("2006-01-02", date); parseErr != nil {
|
||||
return sourceStats{}, nil, fmt.Errorf("invalid date at row %d: %q", stats.Rows+2, date)
|
||||
}
|
||||
if plate == "" {
|
||||
return sourceStats{}, nil, fmt.Errorf("empty plate at row %d", stats.Rows+2)
|
||||
}
|
||||
plates[plate] = struct{}{}
|
||||
dates[date] = struct{}{}
|
||||
stats.Rows++
|
||||
stats.TotalKM += mileage
|
||||
if mileage > 0 {
|
||||
stats.Positive++
|
||||
} else {
|
||||
stats.Zero++
|
||||
}
|
||||
if stats.MinDate == "" || date < stats.MinDate {
|
||||
stats.MinDate = date
|
||||
}
|
||||
if date > stats.MaxDate {
|
||||
stats.MaxDate = date
|
||||
}
|
||||
}
|
||||
stats.PlateCount = len(plates)
|
||||
stats.Vehicles = len(plates)
|
||||
stats.Dates = len(dates)
|
||||
stats.TotalKM = round(stats.TotalKM)
|
||||
return stats, plates, nil
|
||||
}
|
||||
|
||||
func loadMappings(ctx context.Context, conn *sql.Conn) (map[string]mapping, error) {
|
||||
rows, err := conn.QueryContext(ctx, `
|
||||
SELECT plate, vin
|
||||
FROM (
|
||||
SELECT TRIM(plate) AS plate, TRIM(vin) AS vin
|
||||
FROM vehicle
|
||||
WHERE plate IS NOT NULL AND TRIM(plate) <> ''
|
||||
AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
UNION ALL
|
||||
SELECT TRIM(plate) AS plate, TRIM(vin) AS vin
|
||||
FROM vehicle_identifier
|
||||
WHERE enabled = 1 AND plate IS NOT NULL AND TRIM(plate) <> ''
|
||||
AND vin IS NOT NULL AND TRIM(vin) <> ''
|
||||
) mappings`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
result := make(map[string]mapping)
|
||||
for rows.Next() {
|
||||
var plate, vin string
|
||||
if err := rows.Scan(&plate, &vin); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
current, ok := result[plate]
|
||||
if !ok {
|
||||
result[plate] = mapping{VIN: vin}
|
||||
} else if current.VIN != vin {
|
||||
current.Ambiguous = true
|
||||
result[plate] = current
|
||||
}
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func mergeMappingCSV(inputPath string, mappings map[string]mapping) error {
|
||||
file, err := os.Open(inputPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
reader := csv.NewReader(file)
|
||||
header, err := reader.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(header) < 2 {
|
||||
return errors.New("mapping CSV must contain vin,plate")
|
||||
}
|
||||
header[0] = strings.TrimPrefix(header[0], "\uFEFF")
|
||||
if header[0] != "vin" || header[1] != "plate" {
|
||||
return fmt.Errorf("unexpected mapping CSV header: %v", header[:2])
|
||||
}
|
||||
for {
|
||||
record, readErr := reader.Read()
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return readErr
|
||||
}
|
||||
if len(record) < 2 {
|
||||
continue
|
||||
}
|
||||
vin := strings.TrimSpace(record[0])
|
||||
plate := strings.TrimSpace(record[1])
|
||||
if vin == "" || plate == "" {
|
||||
continue
|
||||
}
|
||||
current, ok := mappings[plate]
|
||||
if !ok {
|
||||
mappings[plate] = mapping{VIN: vin}
|
||||
} else if current.VIN != vin {
|
||||
current.Ambiguous = true
|
||||
mappings[plate] = current
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func createStage(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
DROP TEMPORARY TABLE IF EXISTS tmp_g7_mileage_import`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = conn.ExecContext(ctx, `
|
||||
CREATE TEMPORARY TABLE tmp_g7_mileage_import (
|
||||
vin varchar(32) NOT NULL,
|
||||
plate varchar(32) NOT NULL,
|
||||
stat_date date NOT NULL,
|
||||
daily_mileage_km decimal(18,3) NOT NULL,
|
||||
PRIMARY KEY (vin, stat_date),
|
||||
KEY idx_tmp_plate (plate),
|
||||
KEY idx_tmp_date (stat_date)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`)
|
||||
return err
|
||||
}
|
||||
|
||||
func loadStage(ctx context.Context, conn *sql.Conn, inputPath string, resolution plateResolution) (stageStats, error) {
|
||||
reader, closeReader, err := openCSV(inputPath)
|
||||
if err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
defer closeReader()
|
||||
csvReader := csv.NewReader(reader)
|
||||
if _, err := csvReader.Read(); err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
tx, err := conn.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
|
||||
type stageRow struct {
|
||||
VIN, Plate, Date string
|
||||
Mileage float64
|
||||
}
|
||||
batch := make([]stageRow, 0, 1000)
|
||||
flush := func() error {
|
||||
if len(batch) == 0 {
|
||||
return nil
|
||||
}
|
||||
var query strings.Builder
|
||||
query.WriteString("INSERT INTO tmp_g7_mileage_import (vin, plate, stat_date, daily_mileage_km) VALUES ")
|
||||
args := make([]any, 0, len(batch)*4)
|
||||
for index, row := range batch {
|
||||
if index > 0 {
|
||||
query.WriteByte(',')
|
||||
}
|
||||
query.WriteString("(?,?,?,?)")
|
||||
args = append(args, row.VIN, row.Plate, row.Date, row.Mileage)
|
||||
}
|
||||
if _, execErr := tx.ExecContext(ctx, query.String(), args...); execErr != nil {
|
||||
return execErr
|
||||
}
|
||||
batch = batch[:0]
|
||||
return nil
|
||||
}
|
||||
for {
|
||||
record, readErr := csvReader.Read()
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
if readErr != nil {
|
||||
return stageStats{}, readErr
|
||||
}
|
||||
plate := strings.TrimSpace(record[0])
|
||||
vin, ok := resolution.Mapped[plate]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
mileage, _ := strconv.ParseFloat(strings.TrimSpace(record[2]), 64)
|
||||
batch = append(batch, stageRow{
|
||||
VIN: vin, Plate: plate, Date: strings.TrimSpace(record[1]), Mileage: mileage,
|
||||
})
|
||||
if len(batch) == cap(batch) {
|
||||
if err := flush(); err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := flush(); err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return stageStats{}, err
|
||||
}
|
||||
var stats stageStats
|
||||
var nullableSum sql.NullFloat64
|
||||
err = conn.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*), COUNT(DISTINCT vin), COUNT(DISTINCT stat_date),
|
||||
SUM(daily_mileage_km > 0), SUM(daily_mileage_km = 0),
|
||||
SUM(daily_mileage_km)
|
||||
FROM tmp_g7_mileage_import`).Scan(
|
||||
&stats.Rows, &stats.Vehicles, &stats.Dates, &stats.Positive, &stats.Zero, &nullableSum,
|
||||
)
|
||||
if nullableSum.Valid {
|
||||
stats.TotalKM = round(nullableSum.Float64)
|
||||
}
|
||||
return stats, err
|
||||
}
|
||||
|
||||
func inspectExisting(ctx context.Context, conn *sql.Conn) (existingStats, error) {
|
||||
var stats existingStats
|
||||
var nullableSum sql.NullFloat64
|
||||
err := conn.QueryRowContext(ctx, `
|
||||
SELECT
|
||||
SUM(m.vin IS NOT NULL),
|
||||
SUM(m.vin IS NULL),
|
||||
SUM(m.vin IS NOT NULL AND ABS(m.daily_mileage_km - s.daily_mileage_km) > 0.0005),
|
||||
SUM(m.vin IS NOT NULL AND ABS(m.daily_mileage_km - s.daily_mileage_km) <= 0.0005),
|
||||
SUM(m.vin IS NOT NULL AND COALESCE(m.source_id, 0) <> ?),
|
||||
SUM(CASE WHEN m.vin IS NOT NULL THEN m.daily_mileage_km ELSE 0 END)
|
||||
FROM tmp_g7_mileage_import s
|
||||
LEFT JOIN vehicle_daily_mileage m
|
||||
ON m.vin = s.vin AND m.stat_date = s.stat_date AND m.protocol = 'JT808'`, g7SourceID).Scan(
|
||||
&stats.ExistingFinalRows,
|
||||
&stats.NewFinalRows,
|
||||
&stats.ChangedExistingFinalRows,
|
||||
&stats.UnchangedExistingFinalRows,
|
||||
&stats.OtherSourceFinalRows,
|
||||
&nullableSum,
|
||||
)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if nullableSum.Valid {
|
||||
stats.ExistingFinalTotalKM = round(nullableSum.Float64)
|
||||
}
|
||||
err = conn.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_daily_mileage_source c
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = c.vin AND s.stat_date = c.stat_date
|
||||
WHERE c.protocol = 'JT808' AND c.is_selected = 1
|
||||
AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel')`).Scan(
|
||||
&stats.OtherSelectedCandidateRows,
|
||||
)
|
||||
return stats, err
|
||||
}
|
||||
|
||||
func applyImport(ctx context.Context, conn *sql.Conn) (int64, float64, int64, int64, error) {
|
||||
tx, err := conn.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelReadCommitted})
|
||||
if err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE vehicle_daily_mileage_source c
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = c.vin AND s.stat_date = c.stat_date
|
||||
SET c.is_selected = 0
|
||||
WHERE c.protocol = 'JT808'
|
||||
AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel')
|
||||
AND c.is_selected <> 0`); err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("clear other selected candidates: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vehicle_daily_mileage_source (
|
||||
vin, stat_date, protocol, source_key, source_ip, source_endpoint,
|
||||
platform_name, first_total_mileage_km, latest_total_mileage_km,
|
||||
daily_mileage_km, sample_count, first_event_time, latest_event_time,
|
||||
quality_status, quality_reason, is_selected
|
||||
)
|
||||
SELECT
|
||||
s.vin, s.stat_date, 'JT808',
|
||||
CONCAT('JT808:', s.vin, '@PLATFORM:g7s_excel'),
|
||||
'manual-g7s-excel', 'manual-g7s-excel',
|
||||
'G7s Excel Override', NULL, NULL,
|
||||
s.daily_mileage_km, 1,
|
||||
TIMESTAMP(s.stat_date, '00:00:00'),
|
||||
TIMESTAMP(s.stat_date, '23:59:59'),
|
||||
'OK', ?, 1
|
||||
FROM tmp_g7_mileage_import s
|
||||
ON DUPLICATE KEY UPDATE
|
||||
source_ip = VALUES(source_ip),
|
||||
source_endpoint = VALUES(source_endpoint),
|
||||
platform_name = VALUES(platform_name),
|
||||
first_total_mileage_km = NULL,
|
||||
latest_total_mileage_km = NULL,
|
||||
daily_mileage_km = VALUES(daily_mileage_km),
|
||||
sample_count = VALUES(sample_count),
|
||||
first_event_time = VALUES(first_event_time),
|
||||
latest_event_time = VALUES(latest_event_time),
|
||||
quality_status = VALUES(quality_status),
|
||||
quality_reason = VALUES(quality_reason),
|
||||
is_selected = 1`, g7QualityReason); err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("upsert G7 candidates: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
INSERT INTO vehicle_daily_mileage (
|
||||
vin, stat_date, protocol, source_id, daily_mileage_km,
|
||||
latest_total_mileage_km, updated_at
|
||||
)
|
||||
SELECT vin, stat_date, 'JT808', ?, daily_mileage_km, NULL, NOW()
|
||||
FROM tmp_g7_mileage_import
|
||||
ON DUPLICATE KEY UPDATE
|
||||
source_id = VALUES(source_id),
|
||||
daily_mileage_km = VALUES(daily_mileage_km),
|
||||
latest_total_mileage_km = NULL,
|
||||
updated_at = NOW()`, g7SourceID); err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("upsert final mileage: %w", err)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `
|
||||
UPDATE vehicle_data_source
|
||||
SET latest_seen_at = GREATEST(COALESCE(latest_seen_at, '2026-06-30 23:59:59'), '2026-06-30 23:59:59'),
|
||||
first_seen_at = LEAST(COALESCE(first_seen_at, '2026-01-01 00:00:00'), '2026-01-01 00:00:00'),
|
||||
latest_source_endpoint = 'manual-g7s-excel',
|
||||
platform_name = 'G7s Excel Override',
|
||||
source_code = 'g7s_excel',
|
||||
source_kind = 'PLATFORM',
|
||||
trust_priority = 0,
|
||||
enabled = 1,
|
||||
remark = 'Manual GPS override imported from G7s quarterly daily mileage reports for 2026-01-01 through 2026-06-30 on 2026-07-28'
|
||||
WHERE id = ? AND protocol = 'JT808' AND source_ip = 'manual-g7s-excel'`, g7SourceID); err != nil {
|
||||
return 0, 0, 0, 0, fmt.Errorf("update G7 source metadata: %w", err)
|
||||
}
|
||||
var finalRows, candidateRows, otherSelected int64
|
||||
var finalSum sql.NullFloat64
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*), SUM(m.daily_mileage_km)
|
||||
FROM vehicle_daily_mileage m
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = m.vin AND s.stat_date = m.stat_date
|
||||
WHERE m.protocol = 'JT808' AND m.source_id = ?
|
||||
AND ABS(m.daily_mileage_km - s.daily_mileage_km) <= 0.0005`, g7SourceID).Scan(
|
||||
&finalRows, &finalSum,
|
||||
); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_daily_mileage_source c
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = c.vin AND s.stat_date = c.stat_date
|
||||
WHERE c.protocol = 'JT808'
|
||||
AND c.source_key = CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel')
|
||||
AND c.is_selected = 1
|
||||
AND ABS(c.daily_mileage_km - s.daily_mileage_km) <= 0.0005`).Scan(&candidateRows); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
if err := tx.QueryRowContext(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM vehicle_daily_mileage_source c
|
||||
JOIN tmp_g7_mileage_import s
|
||||
ON s.vin = c.vin AND s.stat_date = c.stat_date
|
||||
WHERE c.protocol = 'JT808'
|
||||
AND c.source_key <> CONCAT('JT808:', c.vin, '@PLATFORM:g7s_excel')
|
||||
AND c.is_selected = 1`).Scan(&otherSelected); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
var stageRows int64
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM tmp_g7_mileage_import`).Scan(&stageRows); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
if finalRows != stageRows || candidateRows != stageRows || otherSelected != 0 {
|
||||
return 0, 0, 0, 0, fmt.Errorf(
|
||||
"verification failed before commit: stage=%d final=%d candidates=%d other_selected=%d",
|
||||
stageRows, finalRows, candidateRows, otherSelected,
|
||||
)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, 0, 0, 0, err
|
||||
}
|
||||
return finalRows, round(finalSum.Float64), candidateRows, otherSelected, nil
|
||||
}
|
||||
|
||||
func backupQuery(ctx context.Context, conn *sql.Conn, outputPath, query string) error {
|
||||
rows, err := conn.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.OpenFile(outputPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
gzipWriter := gzip.NewWriter(file)
|
||||
defer gzipWriter.Close()
|
||||
csvWriter := csv.NewWriter(gzipWriter)
|
||||
defer csvWriter.Flush()
|
||||
if err := csvWriter.Write(columns); err != nil {
|
||||
return err
|
||||
}
|
||||
raw := make([]sql.RawBytes, len(columns))
|
||||
pointers := make([]any, len(columns))
|
||||
for index := range raw {
|
||||
pointers[index] = &raw[index]
|
||||
}
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
return err
|
||||
}
|
||||
record := make([]string, len(columns))
|
||||
for index, value := range raw {
|
||||
if value == nil {
|
||||
record[index] = "\\N"
|
||||
} else {
|
||||
record[index] = string(value)
|
||||
}
|
||||
}
|
||||
if err := csvWriter.Write(record); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
csvWriter.Flush()
|
||||
if err := csvWriter.Error(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := gzipWriter.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return file.Sync()
|
||||
}
|
||||
|
||||
func openCSV(inputPath string) (io.Reader, func(), error) {
|
||||
file, err := os.Open(inputPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
closeReader := func() { _ = file.Close() }
|
||||
if !strings.HasSuffix(strings.ToLower(inputPath), ".gz") {
|
||||
return file, closeReader, nil
|
||||
}
|
||||
gzipReader, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
file.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
return gzipReader, func() {
|
||||
_ = gzipReader.Close()
|
||||
_ = file.Close()
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizedDSN(raw string) (string, error) {
|
||||
config, err := mysql.ParseDSN(raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
location, err := time.LoadLocation("Asia/Shanghai")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
config.ParseTime = true
|
||||
config.Loc = location
|
||||
config.Timeout = 10 * time.Second
|
||||
config.ReadTimeout = 30 * time.Minute
|
||||
config.WriteTimeout = 30 * time.Minute
|
||||
config.Params = cloneParams(config.Params)
|
||||
config.Params["charset"] = "utf8mb4"
|
||||
return config.FormatDSN(), nil
|
||||
}
|
||||
|
||||
func cloneParams(input map[string]string) map[string]string {
|
||||
output := make(map[string]string, len(input)+1)
|
||||
for key, value := range input {
|
||||
output[key] = value
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func round(value float64) float64 {
|
||||
return float64(int64(value*100+0.5)) / 100
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dsn := strings.TrimSpace(os.Getenv("MYSQL_DSN"))
|
||||
if dsn == "" {
|
||||
panic("MYSQL_DSN is required")
|
||||
}
|
||||
db, err := sql.Open("mysql", dsn)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer db.Close()
|
||||
for _, table := range []string{
|
||||
"vehicle_data_source",
|
||||
"vehicle_daily_mileage_source",
|
||||
"vehicle_daily_mileage",
|
||||
"vehicle",
|
||||
"vehicle_identifier",
|
||||
} {
|
||||
var name, ddl string
|
||||
if err := db.QueryRow("SHOW CREATE TABLE "+table).Scan(&name, &ddl); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf("=== %s ===\n%s\n", name, ddl)
|
||||
}
|
||||
rows, err := db.Query("SELECT * FROM vehicle_data_source WHERE id = 661200")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if rows.Next() {
|
||||
values := make([]any, len(columns))
|
||||
pointers := make([]any, len(columns))
|
||||
for i := range values {
|
||||
pointers[i] = &values[i]
|
||||
}
|
||||
if err := rows.Scan(pointers...); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
record := make(map[string]any, len(columns))
|
||||
for i, column := range columns {
|
||||
switch value := values[i].(type) {
|
||||
case []byte:
|
||||
record[column] = string(value)
|
||||
default:
|
||||
record[column] = value
|
||||
}
|
||||
}
|
||||
data, _ := json.Marshal(record)
|
||||
fmt.Printf("=== source_661200 ===\n%s\n", data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { FileBlob, SpreadsheetFile } from "@oai/artifact-tool";
|
||||
|
||||
const inputPath =
|
||||
"/Users/lingniu/project/ai-coding/lingniu-vehicle-ingest/outputs/g7-mileage-history-20260724/merged/G7车辆每日里程及里程统计_20220101-20260720.xlsx";
|
||||
const workbook = await SpreadsheetFile.importXlsx(await FileBlob.load(inputPath));
|
||||
const sheets = await workbook.inspect({
|
||||
kind: "sheet",
|
||||
include: "id,name",
|
||||
maxChars: 4000,
|
||||
});
|
||||
const summary = await workbook.inspect({
|
||||
kind: "table",
|
||||
sheetId: "汇总",
|
||||
range: "A1:H25",
|
||||
include: "values,formulas",
|
||||
maxChars: 12000,
|
||||
tableMaxRows: 25,
|
||||
tableMaxCols: 8,
|
||||
});
|
||||
const daily = await workbook.inspect({
|
||||
kind: "table",
|
||||
sheetId: "每日里程非零",
|
||||
range: "A1:D12",
|
||||
include: "values,formulas",
|
||||
maxChars: 6000,
|
||||
tableMaxRows: 12,
|
||||
tableMaxCols: 4,
|
||||
});
|
||||
const errors = await workbook.inspect({
|
||||
kind: "match",
|
||||
searchTerm: "#REF!|#DIV/0!|#VALUE!|#NAME\\?|#N/A",
|
||||
options: { useRegex: true, maxResults: 300 },
|
||||
summary: "post-export formula error scan",
|
||||
});
|
||||
console.log(JSON.stringify({
|
||||
inputPath,
|
||||
sheets: sheets.ndjson,
|
||||
summary: summary.ndjson,
|
||||
daily: daily.ndjson,
|
||||
errors: errors.ndjson,
|
||||
}, null, 2));
|
||||
Reference in New Issue
Block a user